[ { "path": ".coveragerc", "content": "[run]\nbranch = True\nsource = sh\nrelative_files = True\n\n[report]\nexclude_lines =\n pragma: no cover\n if __name__ == .__main__.:\n def __repr__\n" }, { "path": ".flake8", "content": "[flake8]\nmax-line-length = 88\nextend-ignore = E203" }, { "path": ".github/FUNDING.yml", "content": "# These are supported funding model platforms\n\ngithub: [ecederstrand, amoffat]\npatreon: # Replace with a single Patreon username\nopen_collective: # Replace with a single Open Collective username\nko_fi: # Replace with a single Ko-fi username\ntidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel\ncommunity_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry\nliberapay: # Replace with a single Liberapay username\nissuehunt: # Replace with a single IssueHunt username\notechie: # Replace with a single Otechie username\ncustom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']\n" }, { "path": ".github/workflows/main.yml", "content": "# This workflow will install Python dependencies, run tests and converage with a variety of Python versions\n# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions\n\nname: Run tests\n\non:\n pull_request:\n push:\n branches:\n - master\n\njobs:\n lint:\n name: Lint\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n\n - uses: actions/cache@v4\n name: Cache pip directory\n with:\n path: ~/.cache/pip\n key: ${{ runner.os }}-pip-3.9\n\n - uses: actions/cache@v4\n name: Cache poetry deps\n with:\n path: .venv\n key: ${{ runner.os }}-build-${{ hashFiles('poetry.lock') }}-3.9\n\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: 3.9\n\n - name: Install poetry\n run: |\n pip install poetry\n\n - name: Install dependencies\n run: |\n poetry config virtualenvs.in-project true\n poetry install\n\n - name: Lint\n run: |\n poetry run python -m flake8 sh.py tests/*.py\n poetry run black --check --diff sh.py tests/*.py\n poetry run rstcheck README.rst\n poetry run mypy sh.py\n\n test:\n name: Run tests\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n os: [ubuntu-latest]\n python-version: [\"3.8\", \"3.9\", \"3.10\", \"3.11\", \"3.12\"]\n use-select: [0, 1]\n lang: [C, en_US.UTF-8]\n\n steps:\n - uses: actions/checkout@v4\n\n - uses: actions/cache@v4\n name: Cache pip directory\n with:\n path: ~/.cache/pip\n key: ${{ runner.os }}-pip-3.9\n\n - uses: actions/cache@v4\n name: Cache poetry deps\n env:\n cache-name: poetry-deps\n with:\n path: .venv\n key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('poetry.lock') }}-${{ matrix.python-version }}\n\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v5\n with:\n python-version: ${{ matrix.python-version }}\n\n - name: Install poetry\n run: |\n pip install poetry\n\n - name: Install dependencies\n run: |\n poetry config virtualenvs.in-project true\n poetry install\n\n - name: Run tests\n run: |\n SH_TESTS_RUNNING=1 SH_TESTS_USE_SELECT=${{ matrix.use-select }} LANG=${{ matrix.lang }} poetry run coverage run --data-file=coverage.data -a -m pytest\n\n - name: Store coverage\n uses: actions/upload-artifact@v4\n with:\n name: coverage.${{ matrix.use-select }}.${{ matrix.lang }}.${{ matrix.python-version }}\n path: coverage.data\n\n report:\n name: Report Coverage\n needs: test\n runs-on: ubuntu-latest\n steps:\n # required because coveralls complains if we're not in a git dir\n - uses: actions/checkout@v4\n\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: 3.9\n\n - name: Install dependencies\n run: |\n pip install coverage coveralls\n\n - name: Download coverage artifacts\n uses: actions/download-artifact@v4\n with:\n path: coverage-artifacts\n\n - name: Combine coverage\n run: |\n find coverage-artifacts -name coverage.data | xargs coverage combine -a\n\n - name: Report coverage\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n run: |\n coverage report\n coveralls --service=github\n\n deploy:\n name: Deploy\n needs: test\n runs-on: ubuntu-latest\n if: github.ref_name == 'master'\n\n permissions:\n contents: write\n id-token: write\n\n steps:\n - uses: actions/checkout@v4\n\n - name: Get current version\n id: get_version\n run: |\n version=$(sed -n 's/^version = \"\\(.*\\)\"/\\1/p' pyproject.toml)\n echo \"version=$version\" >> \"$GITHUB_OUTPUT\"\n\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: 3.9\n\n - name: Install dependencies\n run: pip install build\n\n - name: Build\n run: python -m build\n\n - name: Tag commit\n run: |\n git tag \"${{steps.get_version.outputs.version}}\" \"${{github.ref_name}}\"\n git push -f origin \"${{steps.get_version.outputs.version}}\"\n\n - name: Get changes\n id: changelog\n run: |\n python dev_scripts/changelog_extract.py ${{ steps.get_version.outputs.version }} \\\n > release_changes.md\n\n - name: Create Release\n id: create-release\n uses: softprops/action-gh-release@v2\n with:\n tag_name: ${{ steps.get_version.outputs.version }}\n name: Release ${{ steps.get_version.outputs.version }}\n body_path: release_changes.md\n draft: false\n prerelease: false\n files: dist/*\n\n - name: Publish\n uses: pypa/gh-action-pypi-publish@release/v1\n" }, { "path": ".gitignore", "content": "__pycache__/\n*.py[co]\n.tox\n.coverage\n/.cache/\n/.venv/\n/build\n/dist\n/docs/build\n/TODO.md\n/htmlcov/" }, { "path": ".python-version", "content": "3.9.16\n" }, { "path": ".readthedocs.yaml", "content": "# .readthedocs.yaml\n# Read the Docs configuration file\n# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details\n\n# Required\nversion: 2\n\n# Set the OS, Python version and other tools you might need\nbuild:\n os: ubuntu-22.04\n tools:\n python: \"3.10\"\n jobs:\n post_create_environment:\n - pip install poetry\n - poetry config virtualenvs.create false\n post_install:\n - poetry install\n - pip install -e .\n\n# Build documentation in the \"docs/\" directory with Sphinx\nsphinx:\n configuration: docs/source/conf.py\n# Optional but recommended, declare the Python requirements required\n# to build your documentation\n# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html\npython:\n install:\n - requirements: docs/requirements.txt\n" }, { "path": ".vscode/tasks.json", "content": "{\n // See https://go.microsoft.com/fwlink/?LinkId=733558\n // for the documentation about the tasks.json format\n \"version\": \"2.0.0\",\n \"tasks\": [\n {\n \"label\": \"Doc builder\",\n \"type\": \"shell\",\n \"command\": \"source ${workspaceFolder}/.venv/bin/activate && find source/ | entr -s 'make clean && make html'\",\n \"options\": {\n \"cwd\": \"${workspaceFolder}/docs\"\n },\n \"problemMatcher\": [],\n \"group\": {\n \"kind\": \"build\"\n },\n \"isBackground\": true,\n \"presentation\": {\n \"echo\": true,\n \"reveal\": \"always\",\n \"focus\": true,\n \"panel\": \"dedicated\",\n \"showReuseMessage\": false,\n \"clear\": true,\n \"close\": true\n }\n }\n ]\n}\n" }, { "path": "CHANGELOG.md", "content": "# Changelog\n\n## 2.2.2 - 2/23/25\n\n- Bugfix where it was impossible to use a signal as an `ok_code` [#699](https://github.com/amoffat/sh/issues/699)\n\n## 2.2.1 - 1/9/25\n\n- Bugfix where `async` and `return_cmd` does not raise exceptions [#746](https://github.com/amoffat/sh/pull/746)\n\n## 2.2.0 - 1/9/25\n\n- `return_cmd` with `await` now works correctly [#743](https://github.com/amoffat/sh/issues/743)\n- Formal support for Python 3.12\n\n## 2.1.0 - 10/8/24\n\n- Add contrib command `sh.contrib.bash` [#736](https://github.com/amoffat/sh/pull/736)\n\n## 2.0.7 - 5/31/24\n\n- Fix `sh.glob` arguments [#708](https://github.com/amoffat/sh/issues/708)\n- Misc modernizations\n\n## 2.0.6 - 8/9/23\n\n- Add back appropriate sdist files [comment](https://github.com/amoffat/sh/commit/89333ae48069a5b445b3535232195b2de6f4648f)\n\n## 2.0.5 - 8/7/23\n\n- Allow nested `with` contexts [#690](https://github.com/amoffat/sh/issues/690)\n- Call correct asyncio function for getting event loop [#683](https://github.com/amoffat/sh/issues/683)\n\n## 2.0.4 - 5/13/22\n\n- Allow `ok_code` to be used with `fg` [#665](https://github.com/amoffat/sh/pull/665)\n- Make sure `new_group` never creates a new session [#675](https://github.com/amoffat/sh/pull/675)\n\n## 2.0.2 / 2.0.3 (misversioned) - 2/13/22\n\n- Performance regression when using a generator with `_in` [#650](https://github.com/amoffat/sh/pull/650)\n- Adding test support for python 3.11\n\n## 2.0.0 - 2/9/22\n\n- Executed commands now return a unicode string by default\n- Removed magical module-like execution contexts [#636](https://github.com/amoffat/sh/issues/636)\n- Added basic asyncio support via `_async`\n- Dropped support for Python < 3.8\n- Bumped default tty size to more standard (24, 80)\n- First argument being a RunningCommand no longer automatically passes it as stdin\n- `RunningCommand.__eq__` no longer has the side effect of executing the command [#518](https://github.com/amoffat/sh/pull/531)\n- `_tee` now supports both \"err\" and \"out\" [#215](https://github.com/amoffat/sh/issues/215)\n- Removed the builtin override `cd` [link](https://github.com/amoffat/sh/pull/584#discussion_r698055681)\n- Altered process launching model to behave more expectedly [#495](https://github.com/amoffat/sh/issues/495)\n- Bugfix where `_no_out` isn't allowed with `_iter=\"err\"` [#638](https://github.com/amoffat/sh/issues/638)\n- Allow keyword arguments to have a list of values [#529](https://github.com/amoffat/sh/issues/529)\n\n## 1.14.3 - 7/17/22\n\n- Bugfix where `Command` was not aware of default call args when wrapping the module [#559](https://github.com/amoffat/sh/pull/573)\n\n## 1.14.1 - 10/24/20\n\n- bugfix where setting `_ok_code` to not include 0, but 0 was the exit code [#545](https://github.com/amoffat/sh/pull/545)\n\n## 1.14.0 - 8/28/20\n\n- `_env` now more lenient in accepting dictionary-like objects [#527](https://github.com/amoffat/sh/issues/527)\n- `None` and `False` arguments now do not pass through to underlying command [#525](https://github.com/amoffat/sh/pull/525)\n- Implemented `find_spec` on the fancy importer, which fixes some Python3.4+ issues [#536](https://github.com/amoffat/sh/pull/536)\n\n## 1.13.1 - 4/28/20\n\n- regression fix if `_fg=False` [#520](https://github.com/amoffat/sh/issues/520)\n\n## 1.13.0 - 4/27/20\n\n- minor Travis CI fixes [#492](https://github.com/amoffat/sh/pull/492)\n- bugfix for boolean long options not respecting `_long_prefix` [#488](https://github.com/amoffat/sh/pull/488)\n- fix deprecation warning on Python 3.6 regexes [#482](https://github.com/amoffat/sh/pull/482)\n- `_pass_fds` and `_close_fds` special kwargs for controlling file descriptor inheritance in child.\n- more efficiently closing inherited fds [#406](https://github.com/amoffat/sh/issues/406)\n- bugfix where passing invalid dictionary to `_env` will cause a mysterious child 255 exit code. [#497](https://github.com/amoffat/sh/pull/497)\n- bugfix where `_in` using 0 or `sys.stdin` wasn't behaving like a TTY, if it was in fact a TTY. [#514](https://github.com/amoffat/sh/issues/514)\n- bugfix where `help(sh)` raised an exception [#455](https://github.com/amoffat/sh/issues/455)\n- bugfix fixing broken interactive ssh tutorial from docs\n- change to automatic tty merging into a single pty if `_tty_in=True` and `_tty_out=True`\n- introducing `_unify_ttys`, default False, which allows explicit tty merging into single pty\n- contrib command for `ssh` connections requiring passwords\n- performance fix for polling output too fast when using `_iter` [#462](https://github.com/amoffat/sh/issues/462)\n- execution contexts can now be used in python shell [#466](https://github.com/amoffat/sh/pull/466)\n- bugfix `ErrorReturnCode` instances can now be pickled\n- bugfix passing empty string or `None` for `_in` hanged [#427](https://github.com/amoffat/sh/pull/427)\n- bugfix where passing a filename or file-like object to `_out` wasn't using os.dup2 [#449](https://github.com/amoffat/sh/issues/449)\n- regression make `_fg` work with `_cwd` again [#330](https://github.com/amoffat/sh/issues/330)\n- an invalid `_cwd` now raises a `ForkException` not an `OSError`.\n- AIX support [#477](https://github.com/amoffat/sh/issues/477)\n- added a `timeout=None` param to `RunningCommand.wait()` [#515](https://github.com/amoffat/sh/issues/515)\n\n## 1.12.14 - 6/6/17\n\n- bugfix for poor sleep performance [#378](https://github.com/amoffat/sh/issues/378)\n- allow passing raw integer file descriptors for `_out` and `_err` handlers\n- bugfix for when `_tee` and `_out` are used, and the `_out` is a tty or pipe [#384](https://github.com/amoffat/sh/issues/384)\n- bugfix where python 3.3+ detected different arg counts for bound method output callbacks [#380](https://github.com/amoffat/sh/issues/380)\n\n## 1.12.12, 1.12.13 - 3/30/17\n\n- pypi readme doc bugfix [PR#377](https://github.com/amoffat/sh/pull/377)\n\n## 1.12.11 - 3/13/17\n\n- bugfix for relative paths to `sh.Command` not expanding to absolute paths [#372](https://github.com/amoffat/sh/issues/372)\n- updated for python 3.6\n- bugfix for SIGPIPE not being handled correctly on pipelined processes [#373](https://github.com/amoffat/sh/issues/373)\n\n## 1.12.10 - 3/02/17\n\n- bugfix for file descriptors over 1024 [#356](https://github.com/amoffat/sh/issues/356)\n- bugfix when `_err_to_out` is True and `_out` is pipe or tty [#365](https://github.com/amoffat/sh/issues/365)\n\n## 1.12.9 - 1/04/17\n\n- added `_bg_exc` for silencing exceptions in background threads [#350](https://github.com/amoffat/sh/pull/350)\n\n## 1.12.8 - 12/16/16\n\n- bugfix for patched glob.glob on python3.5 [#341](https://github.com/amoffat/sh/issues/341)\n\n## 1.12.7 - 12/07/16\n\n- added `_out` and `_out_bufsize` validator [#346](https://github.com/amoffat/sh/issues/346)\n- bugfix for internal stdout thread running when it shouldn't [#346](https://github.com/amoffat/sh/issues/346)\n\n## 1.12.6 - 12/02/16\n\n- regression bugfix on timeout [#344](https://github.com/amoffat/sh/issues/344)\n- regression bugfix on `_ok_code=None`\n\n## 1.12.5 - 12/01/16\n\n- further improvements on cpu usage\n\n## 1.12.4 - 11/30/16\n\n- regression in cpu usage [#339](https://github.com/amoffat/sh/issues/339)\n\n## 1.12.3 - 11/29/16\n\n- fd leak regression and fix for flawed fd leak detection test [#337](https://github.com/amoffat/sh/pull/337)\n\n## 1.12.2 - 11/28/16\n\n- support for `io.StringIO` in python2\n\n## 1.12.1 - 11/28/16\n\n- added support for using raw file descriptors for `_in`, `_out`, and `_err`\n- removed `.close()`ing `_out` handler if FIFO detected\n\n## 1.12.0 - 11/21/16\n\n- composed commands no longer propagate `_bg`\n- better support for using `sys.stdin` and `sys.stdout` for `_in` and `_out`\n- bugfix where `which()` would not stop searching at the first valid executable found in PATH\n- added `_long_prefix` for programs whose long arguments start with something other than `--` [#278](https://github.com/amoffat/sh/pull/278)\n- added `_log_msg` for advanced configuration of log message [#311](https://github.com/amoffat/sh/pull/311)\n- added `sh.contrib.sudo`\n- added `_arg_preprocess` for advanced command wrapping\n- alter callable `_in` arguments to signify completion with falsy chunk\n- bugfix where pipes passed into `_out` or `_err` were not flushed on process end [#252](https://github.com/amoffat/sh/pull/252)\n- deprecated `with sh.args(**kwargs)` in favor of `sh2 = sh(**kwargs)`\n- made `sh.pushd` thread safe\n- added `.kill_group()` and `.signal_group()` methods for better process control [#237](https://github.com/amoffat/sh/pull/237)\n- added `new_session` special keyword argument for controlling spawned process session [#266](https://github.com/amoffat/sh/issues/266)\n- bugfix better handling for EINTR on system calls [#292](https://github.com/amoffat/sh/pull/292)\n- bugfix where with-contexts were not threadsafe [#247](https://github.com/amoffat/sh/issues/195)\n- `_uid` new special keyword param for specifying the user id of the process [#133](https://github.com/amoffat/sh/issues/133)\n- bugfix where exceptions were swallowed by processes that weren't waited on [#309](https://github.com/amoffat/sh/issues/309)\n- bugfix where processes that dupd their stdout/stderr to a long running child process would cause sh to hang [#310](https://github.com/amoffat/sh/issues/310)\n- improved logging output [#323](https://github.com/amoffat/sh/issues/323)\n- bugfix for python3+ where binary data was passed into a process's stdin [#325](https://github.com/amoffat/sh/issues/325)\n- Introduced execution contexts which allow baking of common special keyword arguments into all commands [#269](https://github.com/amoffat/sh/issues/269)\n- `Command` and `which` now can take an optional `paths` parameter which specifies the search paths [#226](https://github.com/amoffat/sh/issues/226)\n- `_preexec_fn` option for executing a function after the child process forks but before it execs [#260](https://github.com/amoffat/sh/issues/260)\n- `_fg` reintroduced, with limited functionality. hurrah! [#92](https://github.com/amoffat/sh/issues/92)\n- bugfix where a command would block if passed a fd for stdin that wasn't yet ready to read [#253](https://github.com/amoffat/sh/issues/253)\n- `_long_sep` can now take `None` which splits the long form arguments into individual arguments [#258](https://github.com/amoffat/sh/issues/258)\n- making `_piped` perform \"direct\" piping by default (linking fds together). this fixes memory problems [#270](https://github.com/amoffat/sh/issues/270)\n- bugfix where calling `next()` on an iterable process that has raised `StopIteration`, hangs [#273](https://github.com/amoffat/sh/issues/273)\n- `sh.cd` called with no arguments no changes into the user's home directory, like native `cd` [#275](https://github.com/amoffat/sh/issues/275)\n- `sh.glob` removed entirely. the rationale is correctness over hand-holding. [#279](https://github.com/amoffat/sh/issues/279)\n- added `_truncate_exc`, defaulting to `True`, which tells our exceptions to truncate output.\n- bugfix for exceptions whose messages contained unicode\n- `_done` callback no longer assumes you want your command put in the background.\n- `_done` callback is now called asynchronously in a separate thread.\n- `_done` callback is called regardless of exception, which is necessary in order to release held resources, for example a process pool\n\n## 1.10 - 12/30/14\n\n- partially applied functions with `functools.partial` have been fixed for `_out` and `_err` callbacks [#160](https://github.com/amoffat/sh/issues/160)\n- `_out` or `_err` being callables no longer puts the running command in the background. to achieve the previous behavior, pass `_bg=True` to your command.\n- deprecated `_with` contexts [#195](https://github.com/amoffat/sh/issues/195)\n- `_timeout_signal` allows you to specify your own signal to kill a timed-out process with. use a constant from the `signal` stdlib module. [#171](https://github.com/amoffat/sh/issues/171)\n- signal exceptions can now be caught by number or name. `SignalException_9 == SignalException_SIGKILL`\n- child processes that timeout via `_timeout` raise `sh.TimeoutException` instead of `sh.SignalExeception_9` [#172](https://github.com/amoffat/sh/issues/172)\n- fixed `help(sh)` from the python shell and `pydoc sh` from the command line. [#173](https://github.com/amoffat/sh/issues/173)\n- program names can no longer be shadowed by names that sh.py defines internally. removed the requirement of trailing underscores for programs that could have their names shadowed, like `id`.\n- memory optimization when a child process's stdin is a newline-delimted string and our bufsize is newlines\n- feature, `_done` special keyword argument that accepts a callback to be called when the command completes successfully [#185](https://github.com/amoffat/sh/issues/185)\n- bugfix for being unable to print a baked command in python3+ [#176](https://github.com/amoffat/sh/issues/176)\n- bugfix for cwd not existing and causing the child process to continue running parent process code [#202](https://github.com/amoffat/sh/issues/202)\n- child process is now guaranteed to exit on exception between fork and exec.\n- fix python2 deprecation warning when running with -3 [PR #165](https://github.com/amoffat/sh/pull/165)\n- bugfix where sh.py was attempting to execute directories [#196](https://github.com/amoffat/sh/issues/196), [PR #189](https://github.com/amoffat/sh/pull/189)\n- only backgrounded processes will ignore SIGHUP\n- allowed `ok_code` to take a `range` object. [#PR 210](https://github.com/amoffat/sh/pull/210/files)\n- added `sh.args` with context which allows overriding of all command defaults for the duration of that context.\n- added `sh.pushd` with context which takes a directory name and changes to that directory for the duration of that with context. [PR #206](https://github.com/amoffat/sh/pull/206)\n- tests now include python 3.4 if available. tests also stop on the first\n python that suite that fails.\n- SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGPIPE, SIGSYS have been added to the list of signals that throw an exception [PR #201](https://github.com/amoffat/sh/pull/201)\n- \"callable\" builtin has been faked for python3.1, which lacks it.\n- \"direct\" option added to `_piped` special keyword argument, which allows sh to hand off a process's stdout fd directly to another process, instead of buffering its stdout internally, then handing it off. [#119](https://github.com/amoffat/sh/issues/119)\n\n## 1.09 - 9/08/13\n\n- Fixed encoding errors related to a system encoding \"ascii\". [#123](https://github.com/amoffat/sh/issues/123)\n- Added exit_code attribute to SignalException and ErrorReturnCode exception classes. [#127](https://github.com/amoffat/sh/issues/127)\n- Making the default behavior of spawned processes to not be explicitly killed when the parent python process ends. Also making the spawned process ignore SIGHUP. [#139](https://github.com/amoffat/sh/issues/139)\n- Made OSX sleep hack to apply to PY2 as well as PY3.\n\n## 1.08 - 1/29/12\n\n- Added SignalException class and made all commands that end terminate by a signal defined in SIGNALS_THAT_SHOULD_THROW_EXCEPTION raise it. [#91](https://github.com/amoffat/sh/issues/91)\n- Bugfix where CommandNotFound was not being raised if Command was created by instantiation. [#113](https://github.com/amoffat/sh/issues/113)\n- Bugfix for Commands that are wrapped with functools.wraps() [#121](https://github.com/amoffat/sh/issues/121]\n- Bugfix where input arguments were being assumed as ascii or unicode, but never as a string in a different encoding.\n- \\_long_sep keyword argument added joining together a dictionary of arguments passed in to a command\n- Commands can now be passed a dictionary of args, and the keys will be interpretted \"raw\", with no underscore-to-hyphen conversion\n- Reserved Python keywords can now be used as subcommands by appending an underscore `_` to them\n\n## 1.07 - 11/21/12\n\n- Bugfix for PyDev when `locale.getpreferredencoding()` is empty.\n- Fixes for IPython3 that involve `sh.` and `sh?`\n- Added `_tee` special keyword argument to force stdout/stderr to store internally and make available for piping data that is being redirected.\n- Added `_decode_errors` to be passed to all stdout/stderr decoding of a process.\n- Added `_no_out`, `_no_err`, and `_no_pipe` special keyword arguments. These are used for long-running processes with lots of output.\n- Changed custom loggers that were created for each process to fixed loggers, so there are no longer logger references laying around in the logging module after the process ends and it garbage collected.\n\n## 1.06 - 11/10/12\n\n- Removed old undocumented cruft of ARG1..ARGN and ARGV.\n- Bugfix where `logging_enabled` could not be set from the importing module.\n- Disabled garbage collection before fork to prevent garbage collection in child process.\n- Major bugfix where cyclical references were preventing process objects (and their associated stdout/stderr buffers) from being garbage collected.\n- Bugfix in RunningCommand and OProc loggers, which could get really huge if a command was called that had a large number of arguments.\n\n## 1.05 - 10/20/12\n\n- Changing status from alpha to beta.\n- Python 3.3 officially supported.\n- Documentation fix. The section on exceptions now references the fact that signals do not raise an exception, even for signals that might seem like they should, e.g. segfault.\n- Bugfix with Python 3.3 where importing commands from the sh namespace resulted in an error related to `__path__`\n- Long-form and short-form options to commands may now be given False to disable the option from being passed into the command. This is useful to pass in a boolean flag that you flip to either True or False to enable or disable some functionality at runtime.\n\n## 1.04 - 10/07/12\n\n- Making `Command` class resolve the `path` parameter with `which` by default instead of expecting it to be resolved before it is passed in. This change shouldn't affect backwards compatibility.\n- Fixing a bug when an exception is raised from a program, and the error output has non-ascii text. This didn't work in Python < 3.0, because .decode()'s default encoding is typically ascii.\n" }, { "path": "CODEOWNERS", "content": "/.github/ @amoffat" }, { "path": "LICENSE.txt", "content": "Copyright (C) 2011-2012 by Andrew Moffat\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n" }, { "path": "MIGRATION.md", "content": "# Migrating from 1._ to 2._\n\nThis document provides an upgrade path from `1.*` to `2.*`.\n\n## `sh.cd` builtin removed\n\nThere is no `sh.cd` command anymore. It was always command implemented in sh, as\nsome systems provide it as a shell builtin, while others have an actual binary.\nBut neither of them persisted the directory change between other `sh` calls,\nwhich is why it was implemented in sh.\n\n### Workaround\n\nIf you were using `sh.cd(dir)`, use the context manager `with sh.pushd(dir)`\ninstead. All of the commands in the managed context will have the correct\ndirectory.\n\n## Removed execution contexts / default arguments\n\nIn `1.*` you could do could spawn a new module from the `sh` module, one which\nhad customized defaults for the special keyword arguments. This module could\nthen be accessed just like `sh`, and you could even import commands from it.\n\nUnfortunately the magic required to make that work was brittle. Also it was not\naligned syntactically with the similar baking concept. We have therefore changed\nthe syntax to align with baking, and also removed the ability to import directly\nfrom this new baked execution context.\n\n### Workaround\n\n```python\nsh2 = sh(_tty_out=False)\nsh2.ls()\n```\n\nBecomes:\n\n```python\nsh2 = sh.bake(_tty_out=False)\nsh2.ls()\n```\n\nAnd\n\n```python\nsh2 = sh.bake(_tty_out=False)\nfrom sh2 import ls\nls()\n```\n\nBecomes:\n\n```python\nsh2 = sh.bake(_tty_out=False)\nls = sh2.ls\nls()\n```\n\n## Return value now a true string\n\nIn `2.*`, the return value of an executed `sh` command has changed (in most cases) from\na `RunningCommand` object to a unicode string. This makes using the output of a command\nmore natural.\n\n### Workaround\n\nTo continue returning a `RunningCommand` object, you must use the `_return_cmd=True`\nspecial keyword argument. You can achieve this on each file with the following code at\nthe top of files that use `sh`:\n\n```python\nimport sh\n\nsh = sh.bake(_return_cmd=True)\n```\n\n## Piping to STDIN\n\nPreviously, if the first argument of a sh command was an instance of `RunningCommand`,\nit was automatically fed into the process's STDIN. This is no longer the case and you\nmust explicitly use `_in=`.\n\n```python\nfrom sh import wc,ls\n\nprint(wc(ls(\"/home/\", \"-l\"), \"-l\"))\n```\n\nBecomes:\n\n```python\nfrom sh import wc,ls\n\nprint(wc(\"-l\", _in=ls(\"/home/\", \"-l\")))\n```\n\nOr:\n\n```python\nfrom sh import wc,ls\n\nprint(wc(\"-l\", _in=ls(\"/home/\", \"-l\", _return_cmd=True)))\n```\n\n### Workaround\n\nNone\n\n## New processes don't launch in new session\n\nIn `1.*`, `_new_session` defaulted to `True`. It now defaults to `False`. The reason\nfor this is that it makes more sense for launched processes to default to being in\nthe process group of the python script, so that they receive SIGINTs correctly.\n\n### Workaround\n\nTo preserve the old behavior:\n\n```python\nimport sh\n\nsh = sh.bake(_new_session=True)\n```\n" }, { "path": "Makefile", "content": "# runs all tests on all envs, in parallel\n.PHONY: test\ntest: build_test_image\n\tdocker run -it --rm amoffat/shtest tox -p\n\n# one test on all envs, in parallel\n.PHONY: test_one\ntest_one: build_test_image\n\tdocker run -it --rm amoffat/shtest tox -p -- $(test)\n\n.PHONY: build_test_image\nbuild_test_image:\n\tdocker build -t amoffat/shtest -f tests/Dockerfile --build-arg cache_bust=951 .\n\n# publishes to PYPI\n.PHONY: release\nrelease:\n\tpoetry publish --dry-run" }, { "path": "README.rst", "content": ".. image:: https://raw.githubusercontent.com/amoffat/sh/master/images/logo-230.png\n :target: https://amoffat.github.com/sh\n :alt: Logo\n\n**If you are migrating from 1.* to 2.*, please see MIGRATION.md**\n\n|\n\n.. image:: https://img.shields.io/pypi/v/sh.svg?style=flat-square\n :target: https://pypi.python.org/pypi/sh\n :alt: Version\n.. image:: https://img.shields.io/pypi/dm/sh.svg?style=flat-square\n :target: https://pypi.python.org/pypi/sh\n :alt: Downloads Status\n.. image:: https://img.shields.io/pypi/pyversions/sh.svg?style=flat-square\n :target: https://pypi.python.org/pypi/sh\n :alt: Python Versions\n.. image:: https://img.shields.io/coveralls/amoffat/sh.svg?style=flat-square\n :target: https://coveralls.io/r/amoffat/sh?branch=master\n :alt: Coverage Status\n\n|\n\nsh is a full-fledged subprocess replacement for Python 3.8 - 3.12, and PyPy\nthat allows you to call *any* program as if it were a function:\n\n.. code:: python\n\n from sh import ifconfig\n print(ifconfig(\"eth0\"))\n\nsh is *not* a collection of system commands implemented in Python.\n\nsh relies on various Unix system calls and only works on Unix-like operating\nsystems - Linux, macOS, BSDs etc. Specifically, Windows is not supported.\n\n`Complete documentation here `_\n\n`Full documentation on a single page for LLM-assisted coding here `_\n\nInstallation\n============\n\n::\n\n $> pip install sh\n\nSupport\n=======\n* `Andrew Moffat `_ - author/maintainer\n* `Erik Cederstrand `_ - maintainer\n\n\nDevelopers\n==========\n\nTesting\n-------\n\nTests are run in a docker container against all supported Python versions. To run, make the following target::\n\n $> make test\n\nTo run a single test::\n\n $> make test='FunctionalTests.test_background' test_one\n\nDocs\n----\n\nTo build the docs, make sure you've run ``poetry install`` to install the dev dependencies, then::\n\n $> cd docs\n $> make html\n\nThis will generate the docs in ``docs/build/html``. You can open the ``index.html`` file in your browser to view the docs.\n\nCoverage\n--------\n\nFirst run all of the tests::\n\n $> SH_TESTS_RUNNING=1 coverage run --source=sh -m pytest\n\nThis will aggregate a ``.coverage``. You may then visualize the report with::\n\n $> coverage report\n\nOr generate visual html files with::\n\n $> coverage html\n\nWhich will create ``./htmlcov/index.html`` that you may open in a web browser.\n" }, { "path": "dev_scripts/changelog_extract.py", "content": "import re\nimport sys\nfrom pathlib import Path\nfrom typing import Iterable\n\nTHIS_DIR = Path(__file__).parent\nCHANGELOG = THIS_DIR.parent / \"CHANGELOG.md\"\n\n\ndef fetch_changes(changelog: Path, version: str) -> Iterable[str]:\n with open(changelog, \"r\") as f:\n lines = f.readlines()\n\n found_a_change = False\n aggregate = False\n for line in lines:\n if line.startswith(f\"## {version}\"):\n aggregate = True\n\n if aggregate:\n if line.startswith(\"-\"):\n line = re.sub(r\"-\\s*\", \"\", line).strip()\n found_a_change = True\n yield line\n elif found_a_change:\n aggregate = False\n\n return changes\n\n\nversion = sys.argv[1].strip()\nchanges = fetch_changes(CHANGELOG, version)\nif not changes:\n exit(1)\n\nfor change in changes:\n print(\"- \" + change)\n" }, { "path": "docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line, and also\n# from the environment for the first two.\nSPHINXOPTS ?= -a -W\nSPHINXBUILD ?= sphinx-build\nSOURCEDIR = source\nBUILDDIR = build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)" }, { "path": "docs/requirements.txt", "content": "toml==0.10.2" }, { "path": "docs/source/conf.py", "content": "# Configuration file for the Sphinx documentation builder.\n\nfrom pathlib import Path\n\nimport toml\n\n_THIS_DIR = Path(__file__).parent\n_REPO = _THIS_DIR.parent.parent\n_PYPROJECT = _REPO / \"pyproject.toml\"\n\npyproject = toml.load(_PYPROJECT)\nnitpicky = True\n\nnitpick_ignore = [\n (\"py:class\", \"Token\"),\n (\"py:class\", \"'Token'\"),\n]\n\nnitpick_ignore_regex = [\n (\"py:class\", r\".*lark.*\"),\n]\n\nversion = pyproject[\"tool\"][\"poetry\"][\"version\"]\nrelease = version\n\n# -- Project information\n\nproject = \"sh\"\ncopyright = \"2023, Andrew Moffat\"\nauthor = \"Andrew Moffat\"\n\n# -- General configuration\n\nextensions = [\n \"sphinx.ext.duration\",\n \"sphinx.ext.doctest\",\n \"sphinx.ext.autodoc\",\n \"sphinx.ext.autosummary\",\n \"sphinx.ext.intersphinx\",\n \"sphinx.ext.todo\",\n]\n\nintersphinx_mapping = {\n \"python\": (\"https://docs.python.org/3/\", None),\n \"sphinx\": (\"https://www.sphinx-doc.org/en/master/\", None),\n \"lark\": (\"https://lark-parser.readthedocs.io/en/latest/\", None),\n \"jinja2\": (\"https://jinja.palletsprojects.com/en/latest/\", None),\n}\nintersphinx_disabled_domains = [\"std\"]\n\ntemplates_path = [\"_templates\"]\n\n# -- Options for HTML output\n\nhtml_theme = \"sphinx_rtd_theme\"\n\n# -- Options for EPUB output\nepub_show_urls = \"footnote\"\n\nautodoc_typehints = \"both\"\n\nadd_module_names = False\n" }, { "path": "docs/source/examples/done.rst", "content": "Here's an example of using :ref:`done` to create a multiprocess pool, where\n``sh.your_parallel_command`` is executed concurrently at no more than 10 at a\ntime:\n\n.. code-block:: python\n\n import sh\n from threading import Semaphore\n\n pool = Semaphore(10)\n\n def done(cmd, success, exit_code):\n pool.release()\n\n def do_thing(arg):\n pool.acquire()\n return sh.your_parallel_command(arg, _bg=True, _done=done)\n\n procs = []\n for arg in range(100):\n procs.append(do_thing(arg))\n\n # essentially a join\n [p.wait() for p in procs]\n" }, { "path": "docs/source/fulldoc.rst", "content": "Full Documentation\n==================\n\nThis single page repeats the full documentation for `sh `, making it easier to put into an LLM's context window. There is nothing on this page that is not mentionned already elsewhere on this site, it's just reorganized as a single page.\n\n.. include:: index.rst\n\n.. include:: tutorials/interacting_with_processes.rst\n.. include:: tutorials/real_time_output.rst\n\n\n.. content linked in index.rst\n.. include:: sections/faq.rst\n.. include:: sections/contrib.rst\n.. include:: sections/sudo.rst\n\n.. content of usage.rst\n.. include:: sections/passing_arguments.rst\n.. include:: sections/exit_codes.rst\n.. include:: sections/redirection.rst\n.. include:: sections/asynchronous_execution.rst\n.. also contains reference to example/done.rst so no need to mention it explicitely\n.. .. include:: examples/done.rst\n.. include:: sections/baking.rst\n.. include:: sections/piping.rst\n.. include:: sections/subcommands.rst\n.. include:: sections/default_arguments.rst\n.. include:: sections/envs.rst\n.. include:: sections/stdin.rst\n.. include:: sections/with.rst\n\n.. content of reference.rst\n.. include:: sections/special_arguments.rst\n.. include:: sections/architecture.rst\n.. include:: sections/command_class.rst\n" }, { "path": "docs/source/index.rst", "content": "\n.. toctree::\n :hidden:\n\n usage\n reference\n\n sections/contrib\n sections/sudo\n\n tutorials\n sections/faq\n\n ref_to_fulldoc\n\n.. image:: images/logo-230.png\n :alt: Logo\n\nsh\n##\n\n.. image:: https://img.shields.io/pypi/v/sh.svg?style=flat-square\n :target: https://pypi.python.org/pypi/sh\n :alt: Version\n.. image:: https://img.shields.io/pypi/dm/sh.svg?style=flat-square\n :target: https://pypi.python.org/pypi/sh\n :alt: Downloads Status\n.. image:: https://img.shields.io/pypi/pyversions/sh.svg?style=flat-square\n :target: https://pypi.python.org/pypi/sh\n :alt: Python Versions\n.. image:: https://img.shields.io/coveralls/amoffat/sh.svg?style=flat-square\n :target: https://coveralls.io/r/amoffat/sh?branch=master\n :alt: Coverage Status\n.. image:: https://img.shields.io/github/stars/amoffat/sh.svg?style=social&label=Star\n :target: https://github.com/amoffat/sh\n :alt: Github\n\nsh is a full-fledged subprocess replacement for Python 3.8+, and PyPy that\nallows you to call any program as if it were a function:\n\n\n.. code-block:: python\n\n\tfrom sh import ifconfig\n\tprint(ifconfig(\"wlan0\"))\n\t\nOutput:\n\n.. code-block:: none\n\n\twlan0\tLink encap:Ethernet HWaddr 00:00:00:00:00:00 \n\t\tinet addr:192.168.1.100 Bcast:192.168.1.255 Mask:255.255.255.0\n\t\tinet6 addr: ffff::ffff:ffff:ffff:fff/64 Scope:Link\n\t\tUP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1\n\t\tRX packets:0 errors:0 dropped:0 overruns:0 frame:0\n\t\tTX packets:0 errors:0 dropped:0 overruns:0 carrier:0\n\t\tcollisions:0 txqueuelen:1000 \n\t\tRX bytes:0 (0 GB) TX bytes:0 (0 GB)\n\t\nNote that these aren't Python functions, these are running the binary commands\non your system by dynamically resolving your ``$PATH``, much like Bash does, and\nthen wrapping the binary in a function. In this way, all the programs on your\nsystem are easily available to you from within Python.\n\nsh relies on various Unix system calls and only works on Unix-like operating\nsystems - Linux, macOS, BSDs etc. Specifically, Windows is not supported.\n\n\nInstallation\n============\n\n.. code-block:: none\n\n pip install sh\n\n\nQuick Reference\n===============\n\nPassing Arguments\n-----------------\n\n.. code-block:: python\n \n sh.ls(\"-l\", \"/tmp\", color=\"never\")\n\n:ref:`Read More `\n\nExit Codes\n----------\n\n.. code-block:: python\n\n try:\n sh.ls(\"/doesnt/exist\")\n except sh.ErrorReturnCode_2:\n print(\"directory doesn't exist\")\n\n:ref:`Read More `\n\nRedirection\n-----------\n\n.. code-block:: python\n\n sh.ls(_out=\"/tmp/dir_contents\")\n\n with open(\"/tmp/dir_contents\", \"w\") as h:\n sh.ls(_out=h)\n\n from io import StringIO\n buf = StringIO()\n sh.ls(_out=buf)\n\n:ref:`Read More `\n\nBaking\n------\n\n.. code-block:: python\n\n my_ls = sh.ls.bake(\"-l\")\n\n # equivalent\n my_ls(\"/tmp\")\n sh.ls(\"-l\", \"/tmp\")\n\n:ref:`Read More `\n\nPiping\n------\n\n.. code-block:: python\n\n sh.wc(\"-l\", _in=sh.ls(\"-1\"))\n\n:ref:`Read More `\n\nSubcommands\n-----------\n\n.. code-block:: python\n\n # equivalent\n sh.git(\"show\", \"HEAD\")\n sh.git.show(\"HEAD\")\n\n:ref:`Read More `\n\n\nBackground Processes\n--------------------\n\n.. code-block:: python\n\n p = sh.find(\"-name\", \"sh.py\", _bg=True)\n # ... do other things ...\n p.wait()\n\n:ref:`Read More `\n\n.. include:: ref_to_fulldoc.rst\n" }, { "path": "docs/source/ref_to_fulldoc.rst", "content": "Single Page\n===========\n\nThe page below repeats the full documentation for `sh ` as a single page, making it easier to put into an LLM's context window.\n\n:doc:`./fulldoc`\n" }, { "path": "docs/source/reference.rst", "content": "Reference\n=========\n\n.. toctree::\n sections/special_arguments\n sections/architecture\n sections/command_class\n" }, { "path": "docs/source/sections/architecture.rst", "content": ".. _architecture:\n\nArchitecture Overview\n#####################\n\nLaunch\n======\n\nWhen it comes time to launch a process\n\n#. Open pipes and/or TTYs STDIN/OUT/ERR.\n#. Open a pipe for communicating pre-exec exceptions from the child to the\n parent.\n#. Open a pipe for child/parent launch synchronization.\n#. :func:`os.fork` a child process.\n\nFrom here, we have two concurrent processes running:\n\nChild\n-----\n\n#. If :ref:`_bg=True ` is set, we ignore :py:data:`signal.SIGHUP`.\n#. If :ref:`_new_session=True `, become a session leader with\n :func:`os.setsid`, else become a process group leader with\n :func:`os.setpgrp`.\n#. Write our session id to the a pipe connected to the parent. This is mainly\n to synchronize with our parent that our session/group logic has finished.\n#. :func:`os.dup2` the file descriptors of our previously-setup TTYs/pipes to\n our STDIN/OUT/ERR file descriptors.\n#. If we're a session leader and our STDIN is a TTY, via :ref:`_tty_in=True\n `, acquire a controlling\n terminal, thereby becoming the controlling process of the session.\n#. Set our GID/UID if we've set a custom one via :ref:`_uid `.\n#. Close all file descriptors greater than STDERR.\n#. Call :func:`os.execv`.\n\nParent\n------\n\n#. Check for any exceptions via the exception pipe connected to the child.\n#. Block and read our child's session id from a pipe connected to the child.\n This synchronizes to us that the child has finished moving between\n sessions/groups and we can now accurately determine its current session id\n and process group.\n#. If we're using a TTY for STDIN, via :ref:`_tty_in=True `, disable\n echoing on the TTY, so that data sent to STDIN is not echoed to STDOUT.\n\nRunning\n=======\n\nAn instance of :ref:`oproc_class` contains two internal threads, one for STDIN,\nand one for STDOUT and STDERR. The purpose of these threads is to handle\nreading/writing to the read/write ends of the process's standard descriptors.\n\nFor example, the STDOUT/ERR thread continually runs :func:`select.select` on the\nmaster ends of the TTYs/pipes connected to STDOUT/ERR, and if they're ready to\nread, reads the available data and aggregates it into the appropriate place.\n\n.. _arch_buffers:\n\nBuffers\n-------\n\nA couple of different buffers must be considered when thinking about how data\nflows through an sh process.\n\nThe first buffer is the buffer associated with the underlying pipe or TTY\nattached to STDOUT/ERR. In the case of a TTY (the default for output), the\nbuffer size is 0, so output is immediate -- a byte written by the process is a\nbyte received by sh. For a pipe, however, the buffer size of the pipe is\ntypically 4-64kb. :manpage:`pipe(2)`.\n\n.. seealso:: FAQ: :ref:`faq_tty_out`\n\nThe second buffer is sh's internal buffers, one for STDOUT and one for STDERR.\nThese buffers aggregate data that has been read from the master end of the TTY\nor pipe attached to the output fd, but before that data is sent along to the\nappropriate output handler (queue, file object, function, etc). Data sits in\nthese buffers until we reach the size specified with :ref:`internal_bufsize`, at\nwhich point the buffer flushes to the output handler.\n\n\nExit\n====\n\nSTDIN Thread Shutdown\n---------------------\n\nOn process completion, our internal threads must complete, as the read end of\nSTDIN, for example, which is connected to the process, is no longer open, so\nwriting to the slave end will no longer work.\n\nSTDOUT/ERR Thread Shutdown\n--------------------------\n\nThe STDOUT/ERR thread is a little more complicated, because although the process\nis not alive, output data may still exist in the pipe/TTY buffer that must be\ncollected. So we essentially just :func:`select.select` on the read ends until\nthey return nothing, indicating that they are complete, then we break out of our\nread loop.\n\n.. _arch_exit_code:\n\nExit Code Processing\n--------------------\n\nThe exit code is obtained from the reaped process. If the process ended from a\nsignal, the exit code is the negative value of that signal. For example,\nSIGKILL would result in an exit code -9.\n\nDone Callback\n-------------\n\nIf specified, the :ref:`done` callback is executed with the :ref:`RunningCommand\n` instance, a boolean indicating success, and the adjusted exit\ncode. After the callback returns, error processing continues. In other words,\nthe done callback is called regardless of success or failure, and there's\nnothing it can do to prevent the :ref:`ErrorReturnCode `\nexceptions from being raised after it completes.\n\n" }, { "path": "docs/source/sections/asynchronous_execution.rst", "content": ".. _async:\n\nAsynchronous Execution\n######################\n\nsh provides a few methods for running commands and obtaining output in a\nnon-blocking fashion.\n\nAsyncIO\n=======\n\n.. versionadded:: 2.0.0\n\nSh supports asyncio on commands with the :ref:`_async=True ` special\nkwarg. This let's you incrementally ``await`` output produced from your command.\n\n.. code-block:: python\n\n\timport asyncio\n\timport sh\n\n\tasync def main():\n\t await sh.sleep(3, _async=True)\n\n\tasyncio.run(main())\n\n.. _iterable:\n\t \nIncremental Iteration\n=====================\n\nYou may also create asynchronous commands by iterating over them with the\n:ref:`iter` special kwarg. This creates an iterable (specifically, a generator)\nthat you can loop over:\n\n.. code-block:: python\n\n\tfrom sh import tail\n\n\t# runs forever\n\tfor line in tail(\"-f\", \"/var/log/some_log_file.log\", _iter=True):\n\t print(line)\n\t \nBy default, :ref:`iter` iterates over STDOUT, but you can change set this\nspecifically by passing either ``\"err\"`` or ``\"out\"`` to :ref:`iter` (instead of\n``True``). Also by default, output is line-buffered, so the body of the loop\nwill only run when your process produces a newline. You can change this by\nchanging the buffer size of the command's output with :ref:`out_bufsize`.\n\n.. note::\n\n If you need a *fully* non-blocking iterator, use :ref:`iter_noblock`. If\n the current iteration would block, :py:data:`errno.EWOULDBLOCK` will be\n returned, otherwise you'll receive a chunk of output, as normal.\n\n.. _background:\n\t\nBackground Processes\n====================\n\nBy default, each running command blocks until completion. If you have a\nlong-running command, you can put it in the background with the :ref:`_bg=True\n` special kwarg:\n\n.. code-block:: python\n\n\t# blocks\n\tsleep(3)\n\tprint(\"...3 seconds later\")\n\t\n\t# doesn't block\n\tp = sleep(3, _bg=True)\n\tprint(\"prints immediately!\")\n\tp.wait()\n\tprint(\"...and 3 seconds later\")\n\nYou'll notice that you need to call :meth:`RunningCommand.wait` in order to exit\nafter your command exits.\n\nCommands launched in the background ignore ``SIGHUP``, meaning that when their\ncontrolling process (the session leader, if there is a controlling terminal)\nexits, they will not be signalled by the kernel. But because sh commands launch\ntheir processes in their own sessions by default, meaning they are their own\nsession leaders, ignoring ``SIGHUP`` will normally have no impact. So the only\ntime ignoring ``SIGHUP`` will do anything is if you use :ref:`_new_session=False\n`, in which case the controlling process will probably be the shell\nfrom which you launched python, and exiting that shell would normally send a\n``SIGHUP`` to all child processes.\n\n.. seealso::\n\n For more information on the exact launch process, see :ref:`architecture`.\n\n.. _callbacks:\n\nOutput Callbacks\n----------------\n\t \nIn combination with :ref:`_bg=True`, sh can use callbacks to process output\nincrementally by passing a callable function to :ref:`out` and/or :ref:`err`.\nThis callable will be called for each line (or chunk) of data that your command\noutputs:\n\n.. code-block:: python\n\n\tfrom sh import tail\n\t\n\tdef process_output(line):\n\t print(line)\n\t\n\tp = tail(\"-f\", \"/var/log/some_log_file.log\", _out=process_output, _bg=True)\n p.wait()\n\nTo control whether the callback receives a line or a chunk, use\n:ref:`out_bufsize`. To \"quit\" your callback, simply return ``True``. This\ntells the command not to call your callback anymore.\n\nThe line or chunk received by the callback can either be of type ``str`` or\n``bytes``. If the output could be decoded using the provided encoding, a\n``str`` will be passed to the callback, otherwise it would be raw ``bytes``.\n\n.. note::\n\n Returning ``True`` does not kill the process, it only keeps the callback\n from being called again. See :ref:`interactive_callbacks` for how to kill a\n process from a callback.\n\t\n.. seealso:: :ref:`red_func`\n\n.. _interactive_callbacks:\n\t \nInteractive callbacks\n---------------------\n\nCommands may communicate with the underlying process interactively through a\nspecific callback signature\nEach command launched through sh has an internal STDIN :class:`queue.Queue`\nthat can be used from callbacks:\n\n.. code-block:: python\n\n\tdef interact(line, stdin):\n\t if line == \"What... is the air-speed velocity of an unladen swallow?\":\n\t stdin.put(\"What do you mean? An African or European swallow?\")\n\t\t\t\n\t elif line == \"Huh? I... I don't know that....AAAAGHHHHHH\":\n\t cross_bridge()\n\t return True\n\t\t\t\n\t else:\n\t stdin.put(\"I don't know....AAGGHHHHH\")\n\t return True\n\t\t\t\n\tp = sh.bridgekeeper(_out=interact, _bg=True)\n p.wait()\n\n.. note::\n\n If you use a queue, you can signal the end of the input (EOF) with ``None``\n\nYou can also kill or terminate your process (or send any signal, really) from\nyour callback by adding a third argument to receive the process object:\n\n.. code-block:: python\n\n\tdef process_output(line, stdin, process):\n\t print(line)\n\t if \"ERROR\" in line:\n\t process.kill()\n\t return True\n\t\n\tp = tail(\"-f\", \"/var/log/some_log_file.log\", _out=process_output, _bg=True)\n\t\nThe above code will run, printing lines from ``some_log_file.log`` until the\nword ``\"ERROR\"`` appears in a line, at which point the tail process will be\nkilled and the script will end.\n\n.. note::\n\n You may also use :meth:`RunningCommand.terminate` to send a SIGTERM, or\n :meth:`RunningCommand.signal` to send a general signal.\n\n\nDone Callbacks\n--------------\n\nA done callback called when the process exits, either normally (through\na success or error exit code) or through a signal. It is *always* called.\n\n.. include:: /examples/done.rst\n" }, { "path": "docs/source/sections/baking.rst", "content": ".. _baking:\n\nBaking\n======\n\nsh is capable of \"baking\" arguments into commands. This is essentially\n`partial application `_,\nlike you might do with :func:`functools.partial`.\n\n.. code-block:: python\n\n\tfrom sh import ls\n\t\n\tls = ls.bake(\"-la\")\n\tprint(ls) # \"/usr/bin/ls -la\"\n\t\n\t# resolves to \"ls -la /\"\n\tprint(ls(\"/\"))\n\nThe idea here is that now every call to ``ls`` will have the \"-la\" arguments\nalready specified. Baking can become very useful when you combine it with\n:ref:`subcommands`:\n\n.. code-block:: python\n\n\tfrom sh import ssh\n\t\n\t# calling whoami on a server. this is a lot to type out, especially if\n\t# you wanted to call many commands (not just whoami) back to back on\n\t# the same server\n\tiam1 = ssh(\"myserver.com\", \"-p 1393\", \"whoami\")\n\t\n\t# wouldn't it be nice to bake the common parameters into the ssh command?\n\tmyserver = ssh.bake(\"myserver.com\", p=1393)\n\t\n\tprint(myserver) # \"/usr/bin/ssh myserver.com -p 1393\"\n\t\n\t# resolves to \"/usr/bin/ssh myserver.com -p 1393 whoami\"\n\tiam2 = myserver.whoami()\n\t\n\tassert(iam1 == iam2) # True!\n\t\nNow that the \"myserver\" callable represents a baked ssh command, you\ncan call anything on the server easily:\n\n.. code-block:: python\n\t\n\t# executes \"/usr/bin/ssh myserver.com -p 1393 tail /var/log/dumb_daemon.log -n 100\"\n\tprint(myserver.tail(\"/var/log/dumb_daemon.log\", n=100))\n\t\n" }, { "path": "docs/source/sections/command_class.rst", "content": "API\n###\n\n\n.. _command_class:\n\nCommand Class\n==============\n\nThe ``Command`` class represents a program that exists on the system and can be\nrun at some point in time. An instance of ``Command`` is never running; an\ninstance of :ref:`RunningCommand ` is spawned for that.\n\nAn instance of ``Command`` can take the form of a manually instantiated object,\nor as an object instantiated by dynamic lookup:\n\n.. code-block:: python\n\n import sh\n\n ls1 = sh.Command(\"ls\")\n ls2 = sh.ls\n \n assert ls1 == ls2\n\n\n.. py:class:: Command(name, search_paths=None)\n\n Instantiates a Command instance, where *name* is the name of a program that\n exists on the user's ``$PATH``, or is a full path itself. If *search_paths*\n is specified, it must be a list of all the paths to look for the program\n name.\n\n .. code-block:: python\n\n from sh import Command\n\n ifconfig = Command(\"ifconfig\")\n ifconfig = Command(\"/sbin/ifconfig\")\n\n\n.. py:method:: Command.bake(*args, **kwargs)\n\n Returns a new Command with ``*args`` and ``**kwargs`` baked in as\n positional and keyword arguments, respectively. Any future calls to the\n returned Command will include ``*args`` and ``**kwargs`` automatically:\n\n .. code-block:: python\n\n from sh import ls\n\n long_ls = ls.bake(\"-l\")\n print(ls(\"/var\"))\n print(ls(\"/tmp\"))\n \n \n .. seealso::\n\n :ref:`baking`\n\n\nSimilar to the above, arguments to the ``sh.Command`` must be separate.\ne.g. the following does not work::\n\n\t\tlscmd = sh.Command(\"/bin/ls -l\")\n\t\ttarcmd = sh.Command(\"/bin/tar cvf /tmp/test.tar /my/home/directory/\")\n\nYou will run into ``CommandNotFound(path)`` exception even when correct full path is specified.\nThe correct way to do this is to :\n\n#. build ``Command`` object using *only* the binary\n#. pass the arguments to the object *when invoking*\n\nas follows::\n\n\t\tlscmd = sh.Command(\"/bin/ls\")\n\t\tlscmd(\"-l\")\n\t\ttarcmd = sh.Command(\"/bin/tar\")\n\t\ttarcmd(\"cvf\", \"/tmp/test.tar\", \"/my/home/directory/\")\n\n.. _running_command:\n\nRunningCommand Class\n====================\n\nThis represents a :ref:`Command ` instance that has been\nor is being executed. It exists as a wrapper around the low-level :ref:`OProc\n`. Most of your interaction with sh objects are with instances of\nthis class. It is only returned if ``_return_cmd=True`` when you execute a command.\n\n.. warning::\n\n Objects of this class behave very much like strings. This was an\n intentional design decision to make the \"output\" of an executing Command\n behave more intuitively.\n\n Be aware that functions that accept real strings only, for example\n ``json.dumps``, will not work on instances of RunningCommand, even though it\n look like a string.\n\n.. _wait_method:\n\n.. py:method:: RunningCommand.wait(timeout=None)\n\n :param timeout: An optional non-negative number to wait for the command to complete. If it doesn't complete by the\n timeout, we raise :ref:`timeout_exc`.\n\n Block and wait for the command to finish execution and obtain an exit code.\n If the exit code represents a failure, we raise the appropriate exception.\n See :ref:`exceptions `.\n\n .. note::\n \n Calling this method multiple times only yields an exception on the first\n call.\n\n This is called automatically by sh unless your command is being executed\n :ref:`asynchronously `, in which case, you may want to call this\n manually to ensure completion.\n\n If an instance of :ref:`Command ` is being used as the stdin\n argument (see :ref:`piping `), :meth:`wait` is also called on that\n instance, and any exceptions resulting from that process are propagated up.\n\n.. py:attribute:: RunningCommand.process\n\n The underlying :ref:`OProc ` instance.\n\n.. py:attribute:: RunningCommand.stdout\n\n A ``@property`` that calls :meth:`wait` and then returns the contents of\n what the process wrote to stdout.\n\n.. py:attribute:: RunningCommand.stderr\n\n A ``@property`` that calls :meth:`wait` and then returns the contents of\n what the process wrote to stderr.\n\n.. py:attribute:: RunningCommand.exit_code\n\n A ``@property`` that calls :meth:`wait` and then returns the process's exit\n code.\n\n.. py:attribute:: RunningCommand.pid\n\n The process id of the process.\n\n.. py:attribute:: RunningCommand.sid\n\n The session id of the process. This will typically be a different session\n than the current python process, unless :ref:`_new_session=False\n ` was specified.\n\n.. py:attribute:: RunningCommand.pgid\n\n The process group id of the process.\n\n.. py:attribute:: RunningCommand.ctty\n\n The controlling terminal device, if there is one.\n\n.. py:method:: RunningCommand.signal(sig_num)\n\n Sends *sig_num* to the process. Typically used with a value from the\n :mod:`signal` module, like :py:data:`signal.SIGHUP` (see :manpage:`signal(7)`).\n\n.. py:method:: RunningCommand.signal_group(sig_num)\n\n Sends *sig_num* to every process in the process group. Typically used with\n a value from the :mod:`signal` module, like :py:data:`signal.SIGHUP` (see\n :manpage:`signal(7)`).\n\n.. py:method:: RunningCommand.terminate()\n\n Shortcut for :meth:`RunningCommand.signal(signal.SIGTERM)\n `.\n\n.. py:method:: RunningCommand.kill()\n\n Shortcut for :meth:`RunningCommand.signal(signal.SIGKILL)\n `.\n\n.. py:method:: RunningCommand.kill_group()\n\n Shortcut for :meth:`RunningCommand.signal_group(signal.SIGKILL)\n `.\n\n.. py:method:: RunningCommand.is_alive()\n\n Returns whether or not the process is still alive.\n\n :rtype: bool\n\n.. _oproc_class:\n\nOProc Class\n===========\n\n.. warning::\n\n Don't use instances of this class directly. It is being documented here for\n posterity, not for direct use.\n\n.. py:method:: OProc.wait()\n\n Block until the process completes, aggregate the output, and populate\n :attr:`OProc.exit_code`.\n\n.. py:attribute:: OProc.stdout\n\n A :class:`collections.deque`, sized to :ref:`_internal_bufsize\n ` items, that contains the process's STDOUT.\n\n.. py:attribute:: OProc.stderr\n\n A :class:`collections.deque`, sized to :ref:`_internal_bufsize\n ` items, that contains the process's STDERR.\n\n.. py:attribute:: OProc.exit_code\n\n Contains the process's exit code, or ``None`` if the process has not yet\n exited.\n\n.. py:attribute:: OProc.pid\n\n The process id of the process.\n\n.. py:attribute:: OProc.sid\n\n The session id of the process. This will typically be a different session\n than the current python process, unless :ref:`_new_session=False\n ` was specified.\n\n.. py:attribute:: OProc.pgid\n\n The process group id of the process.\n\n.. py:attribute:: OProc.ctty\n\n The controlling terminal device, if there is one.\n\n.. py:method:: OProc.signal(sig_num)\n\n Sends *sig_num* to the process. Typically used with a value from the\n :mod:`signal` module, like :py:data:`signal.SIGHUP` (see :manpage:`signal(7)`).\n\n.. py:method:: OProc.signal_group(sig_num)\n\n Sends *sig_num* to every process in the process group. Typically used with\n a value from the :mod:`signal` module, like :py:data:`signal.SIGHUP` (see\n :manpage:`signal(7)`).\n\n.. py:method:: OProc.terminate()\n\n Shortcut for :meth:`OProc.signal(signal.SIGTERM) `.\n\n.. py:method:: OProc.kill()\n\n Shortcut for :meth:`OProc.signal(signal.SIGKILL) `.\n\n.. py:method:: OProc.kill_group()\n\n Shortcut for :meth:`OProc.signal_group(signal.SIGKILL)\n `.\n\nExceptions\n==========\n\n.. _error_return_code:\n\nErrorReturnCode\n---------------\n\n.. py:class:: ErrorReturnCode\n\n This is the base class for, as the name suggests, error return codes. It\n subclasses :py:class:`Exception`.\n\n.. py:attribute:: ErrorReturnCode.full_cmd\n\n The full command that was executed, as a string, so that you can try it on\n the commandline if you wish.\n\n.. py:attribute:: ErrorReturnCode.stdout\n\n The total aggregated STDOUT for the process.\n\n.. py:attribute:: ErrorReturnCode.stderr\n\n The total aggregated STDERR for the process.\n\n.. py:attribute:: ErrorReturnCode.exit_code\n\n The process's adjusted exit code.\n\n .. seealso:: :ref:`arch_exit_code`\n\n\n.. _signal_exc:\n\nSignalException\n---------------\n\nSubclasses :ref:`ErrorReturnCode `. Raised when a command\nreceives a signal that causes it to exit.\n\n.. _timeout_exc:\n\nTimeoutException\n----------------\n\nRaised when a command specifies a non-null :ref:`timeout` and the command times out:\n\n.. code-block:: python\n\n import sh\n\n try:\n sh.sleep(10, _timeout=1)\n except sh.TimeoutException:\n print(\"we timed out, as expected\")\n\nAlso raised when you specify a timeout to :ref:`RunningCommand.wait(timeout=None)`:\n\n.. code-block:: python\n\n import sh\n\n p = sh.sleep(10, _bg=True)\n try:\n p.wait(timeout=1)\n except sh.TimeoutException:\n print(\"we timed out waiting\")\n p.kill()\n\n.. _not_found_exc:\n\nCommandNotFound\n---------------\n\nThis exception is raised in one of the following conditions:\n\n* The program cannot be found on your path.\n* You do not have permissions to execute the program.\n* The program is not marked executable.\n\nThe last two bullets may seem strange, but they fall in line with how a shell like Bash behaves when looking up a\nprogram to execute.\n\n.. note::\n\n ``CommandNotFound`` subclasses ``AttributeError``. As such, the `repr` of it is simply the name of the missing\n attribute.\n\n\nHelper Functions\n================\n\n.. py:function:: which(name, search_paths=None)\n\n Resolves *name* to program's absolute path, or ``None`` if it cannot be\n found. If *search_paths* is list of paths, use that list to look for the\n program, otherwise use the environment variable ``$PATH``.\n\n.. py:function:: pushd(directory)\n\n This function provides a ``with`` context that behaves similar to Bash's\n `pushd\n `_\n by pushing to the provided directory, and popping out of it at the end of\n the context.\n\n .. code-block:: python\n \n import sh\n\n with sh.pushd(\"/tmp\"):\n sh.touch(\"a_file\")\n\n .. note::\n\n It should be noted that we use a reentrant lock, so that different threads\n using this function will have the correct behavior inside of their ``with``\n contexts.\n" }, { "path": "docs/source/sections/contrib.rst", "content": ".. _contrib:\n\nContrib Commands\n################\n\nContrib is an sh sub-module that provides friendly wrappers to useful commands.\nTypically, the commands being wrapped are unintuitive, and the contrib version\nmakes them intuitive.\n\n.. note::\n\n Contrib commands should be considered generally unstable. They will grow and change as the community figures out the\n best interface for them.\n\nCommands\n========\n\nSudo\n----\n\nAllows you to enter your password from the terminal at runtime, or as a string\nin your script.\n\n.. py:function:: sudo(password=None, *args, **kwargs)\n\n Call sudo with ``password``, if specified, else ask the executing user for a\n password at runtime via :func:`getpass.getpass`.\n\n.. seealso:: :ref:`contrib_sudo`\n\n.. _contrib_git:\n\nGit\n---\n\nMany git commands use a pager for output, which can cause an unexpected behavior\nwhen run through sh. To account for this, the contrib version sets\n``_tty_out=False`` for all git commands.\n\n.. py:function:: git(*args, **kwargs)\n\n Call git with STDOUT connected to a pipe, instead of a TTY.\n\n.. code-block:: python\n\n from sh.contrib import git\n repo_log = git.log()\n\n.. seealso:: :ref:`faq_tty_out` and :ref:`faq_color_output`\n\n.. _contrib_ssh:\n\nSSH\n---\n\n.. versionadded:: 1.13.0\n\nSSH password-based logins :ref:`can be a pain `. This contrib command performs all of the ugly setup and\nprovides a clean interface to using SSH.\n\n.. py:function:: ssh(interact=None, password=None, prompt_match=None, login_success=None, *args, **kwargs)\n\n :param interact: A callback to handle SSH session interaction *after* login is successful. Required.\n :param password: A password string or a function that returns a password string. Optional. If not provided, :func:`getpass.getpass` is used.\n :param prompt_match: The string to match in order to determine when to provide SSH with the password. Or a function\n that matches on the output. Optional.\n :param login_success: A function to determine if SSH login is successful. Optional.\n\nThe ``interact`` parameter takes a callback with a signature that is slightly different to the function callbacks for\n:ref:`redirection `:\n\n.. py:function:: fn(content, stdin_queue)\n \n :param content: An instance of an ephemeral :ref:`SessionContent ` class whose job is to hold the\n characters that the SSH session has written to STDOUT.\n :param stdin_queue: A :class:`queue.Queue` object to communicate with STDIN programmatically.\n\n``password`` can be simply a string that will be used to type the password. If it's not provided, it will be read from STDIN\nat runtime via :func:`getpass.getpass`. It can also be a callable that returns the password string.\n\n``prompt_match`` is a string to match before the contrib command will provide the SSH process with the password. It is\noptional, and if left unspecified, will default to \"password: \". It can also be a callable that is called on a\n:ref:`SessionContent ` instance and returns ``True`` or ``False`` for a match.\n\n``login_success`` is a function that takes a :ref:`SessionContent ` object and returns a boolean for\nwhether or not a successful login occurred. It is optional, and if unspecified, simply evaluates to ``True``, meaning\nany password submission results in a successful login (obviously not always correct). It is recommended that you specify\nthis.\n\n.. _session_content:\n\n.. py:class:: SessionContent()\n\n This class contains a record lines and characters written to the SSH processes's STDOUT. It should be all you need\n from the callbacks to determine how to interact with the SSH process.\n\n.. py:attribute:: SessionContent.chars\n \n :type: :class:`collections.deque`\n \n The previous 50,000 characters.\n\n.. py:attribute:: SessionContent.lines\n \n :type: :class:`collections.deque`\n \n The previous 5,000 lines.\n\n.. py:attribute:: SessionContent.line_chars\n \n :type: list\n\n The characters in the line currently being aggregated.\n\n.. py:attribute:: SessionContent.cur_line\n \n :type: str\n\n A string of the line currently being aggregated.\n\n.. py:attribute:: SessionContent.last_line\n \n :type: str\n\n The previous line.\n\n.. py:attribute:: SessionContent.cur_char\n \n :type: str\n\n The currently written character.\n\n\n.. _contrib_bash:\n\nBash\n---\n\nOften users may find themselves having to run bash commands directly, whether due\nto commands having special characters (e.g. dash, or dot) or other reasons. \nThis can lead into recurrently having to bake the ``bash`` command to call it directly. To \naccount for this, the contrib version provides a ``bash`` command baked in:\n\n.. py:function:: bash(*args, **kwargs)\n\n Call bash with the prefix of \"bash -c [...]\".\n\n.. code-block:: python\n\n from sh.contrib import bash\n\n # Calling commands directly\n bash.ls() # equivallent to \"bash -c ls\"\n\n # Or adding the full commands\n bash(\"command-with-dashes args\")\n\n\nExtending\n=========\n\nFor developers.\n\nTo extend contrib, simply decorate a function in sh with the ``@contrib``\ndecorator, and pass in the name of the command you wish to shadow to the\ndecorator. This method must return an instance of :ref:`Command\n`:\n\n.. code-block:: python\n\n @contrib(\"ls\")\n def my_ls(original):\n ls = original.bake(\"-l\")\n return ls\n\nNow you can run your custom contrib command from your scripts, and you'll be\nusing the command returned from your decorated function:\n\n\n.. code-block:: python\n\n from sh.contrib import ls\n\n # executing: ls -l\n print(ls(\"/\"))\n\nFor even more flexibility, you can design your contrib command to rewrite its\noptions based on *executed* arguments. For example, say you only wish to set a\ncommand's argument if another argument is set. You can accomplish it like this:\n\n.. code-block:: python\n\n @contrib(\"ls\")\n def my_ls(original):\n def process(args, kwargs):\n if \"-a\" in args:\n args.append(\"-L\")\n return args, kwargs\n\n ls = original.bake(\"-l\")\n return ls, process\n\nReturning a process function along with the command will tell sh to use that\nfunction to preprocess the arguments at execution time using the\n:ref:`_arg_preprocess ` special kwarg.\n" }, { "path": "docs/source/sections/default_arguments.rst", "content": ".. _default_arguments:\n\nDefault Arguments\n=================\n\nMany times, you want to override the default arguments of all commands launched\nthrough sh. For example, suppose you want the output of all commands to be\naggregated into a :class:`io.StringIO` buffer. The naive way would be this:\n\n.. code-block:: python\n\n import sh\n from io import StringIO\n\n buf = StringIO()\n\n sh.ls(\"/\", _out=buf)\n sh.whoami(_out=buf)\n sh.ps(\"auxwf\", _out=buf)\n\nClearly, this gets tedious quickly. Fortunately, we can create execution\ncontexts that allow us to set default arguments on all commands spawned from\nthat context:\n\n.. code-block:: python\n\n import sh\n from io import StringIO\n\n buf = StringIO()\n sh2 = sh.bake(_out=buf)\n\n sh2.ls(\"/\")\n sh2.whoami()\n sh2.ps(\"auxwf\")\n\nNow, anything launched from ``sh2`` will send its output to the ``StringIO``\ninstance ``buf``." }, { "path": "docs/source/sections/envs.rst", "content": ".. _environments:\n\nEnvironments\n============\n\nThe :ref:`_env ` special kwarg allows you to pass a dictionary of\nenvironment variables and their corresponding values:\n\n.. code-block:: python\n\n\timport sh\n\tsh.google_chrome(_env={\"SOCKS_SERVER\": \"localhost:1234\"})\n\t\n\n:ref:`_env ` replaces your process's environment completely. Only the\nkey-value pairs in :ref:`_env ` will be used for its environment. If you\nwant to add new environment variables for a process *in addition to* your\nexisting environment, try something like this:\n\n.. code-block:: python\n\n import os\n import sh\n \n new_env = os.environ.copy()\n new_env[\"SOCKS_SERVER\"] = \"localhost:1234\"\n \n sh.google_chrome(_env=new_env)\n\n.. seealso::\n\n To make an environment apply to all sh commands look into\n :ref:`default_arguments`.\n" }, { "path": "docs/source/sections/exit_codes.rst", "content": ".. _exit_codes:\n\nExit Codes & Exceptions\n=======================\n\nNormal processes exit with exit code 0. This can be seen from\n:attr:`RunningCommand.exit_code`:\n\n.. code-block:: python\n\n\toutput = ls(\"/\", _return_cmd=True)\n\tprint(output.exit_code) # should be 0\n\t\nIf a process terminates, and the exit code is not 0, an exception is generated\ndynamically. This lets you catch a specific return code, or catch all error\nreturn codes through the base class :class:`ErrorReturnCode`:\n\n.. code-block:: python\n\n try:\n print(ls(\"/some/non-existant/folder\"))\n except ErrorReturnCode_2:\n print(\"folder doesn't exist!\")\n create_the_folder()\n except ErrorReturnCode:\n print(\"unknown error\")\n\nYou can also customize which exit codes indicate an error with :ref:`ok_code`. For example:\n\n.. code-block:: python\n\n for i in range(10):\n \tsh.grep(\"string to check\", f\"file_{i}.txt\", _ok_code=(0, 1))\n\nwhere the :ref:`ok_code` makes a failure to find a match a no-op.\n\nSignals\n-------\n\nSignals are raised whenever your process terminates from a signal. The\nexception raised in this situation is :ref:`signal_exc`, which subclasses\n:ref:`error_return_code`.\n\n.. code-block:: python\n\n try:\n p = sh.sleep(3, _bg=True)\n p.kill()\n except sh.SignalException_SIGKILL:\n print(\"killed\")\n\nThis behavior could be blocked by appending the negative value of the signal to\n:ref:`ok_code`. All signals that raises :ref:`signal_exc` are ``[SIGABRT, \nSIGBUS, SIGFPE, SIGILL, SIGINT, SIGKILL, SIGPIPE, SIGQUIT, SIGSEGV, SIGTERM, \nSIGTERM]``.\n\n.. note::\n\n You can catch :ref:`signal_exc` by using either a number or a signal name.\n For example, the following two exception classes are equivalent:\n\n .. code-block:: python\n\n assert sh.SignalException_SIGKILL == sh.SignalException_9\n" }, { "path": "docs/source/sections/faq.rst", "content": ".. _faq:\n\nFAQ\n===\n\nHow do I execute a bash builtin?\n--------------------------------\n\n.. code-block:: python\n\n import sh\n\n sh.bash(\"-c\", \"your_builtin\")\n\nOr\n\n.. code-block:: python\n\n import sh\n\n builtins = sh.bash.bake(\"-c\")\n builtins(\"your_builtin\")\n\n\nWill Windows be supported?\n--------------------------\n\nThere are no plans to support Windows.\n\n.. _faq_append:\n\nHow do I append output to a file?\n---------------------------------\n\nUse a file object opened in the mode you desire:\n\n.. code-block:: python\n\n import sh\n\n h = open(\"/tmp/output\", \"a\")\n\n sh.ls(\"/dir1\", _out=h)\n sh.ls(\"/dir2\", _out=h)\n\n.. _faq_color_output:\n\nWhy does my command's output have color?\n----------------------------------------\n\nTypically the reason for this is that your program detected that its STDOUT was\nconnected to a TTY, and therefore decided to print color escape sequences in its\noutput. The typical solution is to use :ref:`_tty_out=False `, which\nwill force a pipe to be connected to STDOUT, and probably change the behavior of\nthe program.\n\n.. seealso::\n\n Git is one of the programs that makes extensive use of terminal colors (as\n well as pagers) in its output, so we added :ref:`a contrib version\n ` for convenience.\n\n.. _faq_tty_out:\n\nWhy is _tty_out=True the default?\n---------------------------------\n\nThis was a design decision made for two reasons:\n\n1. To make programs behave in the same way as seen on the commandline.\n2. To provide better buffering control than pipes allow.\n\nFor #1, we want sh to produce output that is identical to what the user sees\nfrom the commandline, because that's typically the only output they ever see\nfrom their command. This makes the output easy to understand.\n\nFor #2, using a TTY for STDOUT allows us to precisely control the buffering of a\ncommand's output to sh's internal code.\n\n.. seealso:: :ref:`arch_buffers`\n\nOf course, there are some gotchas with TTY STDOUT. One of them is commands that\nuse a pager, for example:\n\n.. code-block:: python\n\n import sh\n print(sh.git.log())\n\n\nThis will sometimes raise a ``SignalException_SIGPIPE``. The reason is because\n``git log`` detects a TTY STDOUT and forks the system’s pager (typically\n``less``) to handle the output. The pager checks for a controlling terminal,\nand, finding none, exits with exit code 1. The exit of the pager means no more\nreaders on ``git log``’s output, and thus a ``SIGPIPE`` is received.\n\nOne solution to the ``git log`` problem above is simply to use\n``_tty_out=False``. Another option, specifically for git, is to use the\n``git --no-pager`` option:\n\n.. code-block:: python\n\n import sh\n print(sh.git('--no-pager', 'log'))\n\n\nWhy doesn't \"*\" work as a command argument?\n-------------------------------------------\n\nGlob expansion is a feature of a shell, like Bash, and is performed by the shell\nbefore passing the results to the program to be exec'd. Because sh is not a\nshell, but rather tool to execute programs directly, we do not handle glob\nexpansion like a shell would.\n\nSo in order to use ``\"*\"`` like you would on the commandline, pass it into\n:func:`glob.glob` first:\n\n.. code-block:: python\n\n import sh\n import glob\n sh.ls(glob.glob(\"*.py\"))\n\n\n.. _faq_path:\n\nHow do I call a program that isn't in ``$PATH``?\n------------------------------------------------\n\nUse the :meth:`Command` constructor to instantiate an instance of Command\ndirectly, then execute that:\n\n.. code-block:: python\n\n import sh\n cmd = sh.Command(\"/path/to/command\")\n cmd(\"-v\", \"arg1\")\n\nHow do I execute a program with a dash in its name?\n---------------------------------------------------\n\nIf it's in your ``$PATH``, substitute the dash for an underscore:\n\n.. code-block:: python\n\n import sh\n sh.google_chrome(\"http://google.com\")\n\nThe above will run ``google-chrome http://google.com``\n\n.. note::\n\n If a program named ``google_chrome`` exists on your system, that will be\n called instead. In that case, in order to execute the program with a dash\n in the name, you'll have to use the method described :ref:`here.\n `\n\n.. _faq_special:\n\nHow do I execute a program with a special character in its name?\n----------------------------------------------------------------\n\nPrograms with non-alphanumeric, non-dash characters in their names cannot be\nexecuted directly as an attribute on the sh module. For example, **this will not\nwork:**\n\n.. code-block:: python\n\n import sh\n sh.mkfs.ext4()\n\nThe reason should be fairly obvious. In Python, characters like ``.`` have\nspecial meaning, in this case, attribute access. What sh is trying to do in the\nabove example is find the program \"mkfs\" (which may or may not exist) and then\nperform a :ref:`subcommand lookup ` with the name \"ext4\". In other\nwords, it will try to call ``mkfs`` with the argument ``ext4``, which is\nprobably not what you want.\n\nThe workaround is instantiating the :ref:`Command Class ` with\nthe string of the program you're looking for:\n\n.. code-block:: python\n\n import sh\n mkfsext4 = sh.Command(\"mkfs.ext4\")\n mkfsext4() # run it\n\n.. _faq_pipe_syntax:\n\n\nWhy not use ``|`` to pipe commands?\n-----------------------------------\n\nI prefer the syntax of sh to resemble function composition instead of a\npipeline. One of the goals of sh is to make executing processes more like\ncalling functions, not making function calls more like Bash.\n\nWhy isn't piping asynchronous by default?\n-----------------------------------------\n\nThere is a non-obvious reason why async piping is not possible by default.\nConsider the following example:\n\n.. code-block:: python\n\n import sh\n\n sh.cat(sh.echo(\"test\\n1\\n2\\n3\\n\"))\n\nWhen this is run, ``sh.echo`` executes and finishes, then the entire output\nstring is fed into ``sh.cat``. What we would really like is each\nnewline-delimited chunk to flow to ``sh.cat`` incrementally.\n\nBut for this example to flow data asynchronously from echo to cat, the echo\ncommand would need to *not block.* But how can the inner command know the\ncontext of its execution, to know to block sometimes but not other times? It\ncan't know that without something explicit.\n\nThis is why the :ref:`piped` special kwarg was introduced. By default, commands\nexecuted block until they are finished, so in order for an inner command to not\nblock, ``_piped=True`` signals to the inner command that it should not block.\nThis way, the inner command starts running, then very shortly after, the outer\ncommand starts running, and both are running simultaneously. Data can then flow\nfrom the inner command to the outer command asynchronously:\n\n.. code-block:: python\n\n import sh\n\n sh.cat(sh.echo(\"test\\n1\\n2\\n3\\n\", _piped=True))\n\nAgain, this example is contrived -- a better example would be a long-running\ncommand that produces a lot of output that you wish to pipe through another\nprogram incrementally.\n\nHow do I run a command and connect it to sys.stdout and sys.stdin?\n------------------------------------------------------------------\n\nThere are two ways to do this\n\n.. seealso:: :ref:`fg`\n\nYou can use :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` as\narguments to :ref:`in`, :ref:`out`, :ref:`err`, respectively, and it *should*\nmostly work as expected:\n\n.. code-block:: python\n\n import sh\n import sys\n sh.your_command(_in=sys.stdin, _out=sys.stdout)\n\nThere are a few reasons why this probably won't work. The first reason is that\n:data:`sys.stdin` is probably a controlling TTY (attached to the shell that\nlaunched the python process), and probably not set in raw mode\n:manpage:`termios(3)`, which means that, among other things, input is buffered\nby newlines.\n\nThe real solution is to use :ref:`_fg=True `:\n\n.. code-block:: python\n\n import sh\n sh.top(_fg=True)\n\n\n.. _faq_separate_args:\n\nWhy do my arguments need to be separate strings?\n------------------------------------------------\n\nThis confuses many new sh users. They want to do something like this and expect\nit to just work:\n\n.. code-block:: python\n\n from sh import tar\n tar(\"cvf /tmp/test.tar /my/home/directory\")\n\nBut instead they'll get a confusing error message:\n\n.. code-block:: none\n\n RAN: '/bin/tar cvf /tmp/test.tar /my/home/directory'\n\n STDOUT:\n\n STDERR:\n /bin/tar: Old option 'f' requires an argument.\n Try '/bin/tar --help' or '/bin/tar --usage' for more information.\n\nThe reason why they expect it to work is because shells, like Bash, automatically\nparse your commandline and break up arguments for you, before sending them to\nthe binary. They have a complex set of rules (some of which are represented by\n:mod:`shlex`) to take a single string of a command and arguments and separate\nthem.\n\nEven if we wanted to implement this in sh (which we don't), it would hurt the\nability for users to parameterize parts of their arguments. They would have to\nuse string interpolation, which would be ugly and error prone:\n\n.. code-block:: python\n\n from sh import tar\n tar(\"cvf %s %s\" % (\"/tmp/tar1.tar\", \"/home/oh no a space\")\n\nIn the above example, ``\"/home/oh\"``, ``\"no\"``, ``\"a\"``, and ``\"space\"`` would\nall be separate arguments to tar, causing the program to behave unexpectedly.\nBasically every command with parameterized arguments would need to expect\ncharacters that could break the parser.\n\n.. _faq_arg_ordering:\n\nHow do I order keyword arguments?\n---------------------------------\n\nTypically this question gets asked when a user is trying to execute something\nlike the following commandline:\n\n.. code-block:: none\n\n my-command --arg1=val1 arg2 --arg3=val3\n\nThis is usually the first attempt that they make:\n\n.. code-block:: python\n\n sh.my_command(arg1=\"val1\", \"arg2\", arg3=\"val3\")\n\nThis doesn't work because, in Python, position arguments, like ``arg2`` cannot\ncome after keyword arguments.\n\nFurthermore, it is entirely possible that ``--arg3=val3`` comes before\n``--arg1=val1``. The reason for this is that a function's ``**kwargs`` is an\nunordered mapping, and so key-value pairs are not guaranteed to resolve to a\nspecific order.\n\nSo the solution here is to forego the usage of the keyword argument\n*convenience*, and just use raw ordered arguments:\n\n.. code-block:: python\n\n sh.my_command(\"--arg1=val1\", \"arg2\", \"--arg3=val3\")\n\n.. _faq_pylint:\n\nHow to disable pylint E1101 no-member errors?\n---------------------------------------------\n\nPylint complains with E1101 no-member to almost all ``sh.command`` invocations,\nbecause it doesn't know, that these members are generated dynamically.\nStarting with Pylint 1.6 these messages can be suppressed using `generated-members `_ option.\n\nJust add following lines to ``pylintrc``::\n\n [TYPECHECK]\n generated-members=sh\n\n\nHow do I patch sh in my tests?\n------------------------------\n\nsh can be patched in your tests the typical way, with\n:func:`unittest.mock.patch`:\n\n.. code-block:: python\n\n from unittest.mock import patch\n import sh\n\n def get_something():\n return sh.pwd()\n\n @patch(\"sh.pwd\", create=True)\n def test_something(pwd):\n pwd.return_value = \"/\"\n assert get_something() == \"/\"\n\nThe important thing to note here is that ``create=True`` is set. This is\nrequired because sh is a bit magical and ``patch`` will fail to find the ``pwd``\ncommand as an attribute on the sh module.\n\nYou may also patch the :class:`Command` class:\n\n.. code-block:: python\n\n from unittest.mock import patch\n import sh\n\n def get_something():\n pwd = sh.Command(\"pwd\")\n return pwd()\n\n @patch(\"sh.Command\")\n def test_something(Command):\n Command().return_value = \"/\"\n assert get_something() == \"/\"\n\nNotice here we do not need ``create=True``, because :class:`Command` is not an\nautomatically generated object on the sh module (it actually exists).\n\n\nWhy is sh just a single file?\n-----------------------------\n\nWhen sh was first written, the design decision was made to make it a single-file\nmodule. This has pros and cons:\n\nCons:\n\n- Auditing the code is more challenging\n- Without file-enforced structure, adding more features and abstractions makes\n the code harder to follow\n- Cognitively, it feels cluttered\n\nPros:\n\n- Can be used easily on systems without Python package managers\n- Can be embedded/bundled together with other software more easily\n- Cognitively, it feels more self-contained\n\nIn my mind, because the primary target audience of sh users is generally more\nscrappy devops, systems people, or people just trying to stitch together some\nclunky system programs, the listed pros weigh a little more heavily than the\ncons. Sacrificing some development advantages to give those users a more\nflexible tool is a win to me.\n\nDown the road, the development disadvantages of a single file can be solved with\nadditional development tools, for example, with a tool that compiles multiple\nmodules into the single sh.py file. Realistically, though, sh is pretty mature,\nso I don't see it growing much more in complexity or code size.\n\nHow do I see the commands sh is running?\n----------------------------------------\n\nUse logging:\n\n.. code-block:: python\n\n import logging\n import sh\n\n logging.basicConfig(level=logging.INFO)\n sh.ls()\n\n.. code-block:: none\n\n INFO:sh.command:: starting process\n INFO:sh.command:: process started\n INFO:sh.command:: process completed\n ...\n" }, { "path": "docs/source/sections/passing_arguments.rst", "content": ".. _passing_arguments:\n\nPassing Arguments\n=================\n\nWhen passing multiple arguments to a command, each argument *must* be a separate\nstring:\n\n.. code-block:: python\n\n from sh import tar\n tar(\"cvf\", \"/tmp/test.tar\", \"/my/home/directory/\")\n\nThis *will not work*:\n\n.. code-block:: python\n\n from sh import tar\n tar(\"cvf /tmp/test.tar /my/home/directory\")\n\t\n.. seealso:: :ref:`faq_separate_args`\n\n\nKeyword Arguments\n-----------------\n\nsh supports short-form ``-a`` and long-form ``--arg`` arguments as\nkeyword arguments:\n\n.. code-block:: python\n\n\t# resolves to \"curl http://duckduckgo.com/ -o page.html --silent\"\n\tcurl(\"http://duckduckgo.com/\", o=\"page.html\", silent=True)\n\t\n\t# or if you prefer not to use keyword arguments, this does the same thing:\n\tcurl(\"http://duckduckgo.com/\", \"-o\", \"page.html\", \"--silent\")\n\t\n\t# resolves to \"adduser amoffat --system --shell=/bin/bash --no-create-home\"\n\tadduser(\"amoffat\", system=True, shell=\"/bin/bash\", no_create_home=True)\n\t\n\t# or\n\tadduser(\"amoffat\", \"--system\", \"--shell\", \"/bin/bash\", \"--no-create-home\")\n\n.. seealso:: :ref:`faq_arg_ordering`\n" }, { "path": "docs/source/sections/piping.rst", "content": ".. _piping:\n\nPiping\n======\n\nBasic\n-----\n\nBash style piping is performed using function composition. Just pass one\ncommand as the input to another's ``_in`` argument, and sh will send the output of\nthe inner command to the input of the outer command:\n\n.. code-block:: python\n\n\t# sort this directory by biggest file\n\tprint(sort(\"-rn\", _in=du(glob(\"*\"), \"-sb\")))\n\t\n\t# print(the number of folders and files in /etc\n\tprint(wc(\"-l\", _in=ls(\"/etc\", \"-1\")))\n\n.. note::\n\n This basic piping does not flow data through asynchronously; the inner\n command blocks until it finishes, before sending its data to the outer\n command.\n\t\nBy default, any command that is piping another command in waits for it to\ncomplete. This behavior can be changed with the :ref:`_piped ` special\nkwarg on the command being piped, which tells it not to complete before sending\nits data, but to send its data incrementally. Read ahead for examples of this.\n\n.. _advanced_piping:\n\nAdvanced\n--------\n\nBy default, all piped commands execute sequentially. What this means is that the\ninner command executes first, then sends its data to the outer command:\n\n.. code-block:: python\n\n\tprint(wc(\"-l\", _in=ls(\"/etc\", \"-1\")))\n\t\nIn the above example, ``ls`` executes, gathers its output, then sends that output\nto ``wc``. This is fine for simple commands, but for commands where you need\nparallelism, this isn't good enough. Take the following example:\n\n.. code-block:: python\n\n\tfor line in tr(_in=tail(\"-f\", \"test.log\"), \"[:upper:]\", \"[:lower:]\", _iter=True):\n\t print(line)\n\t\n**This won't work** because the ``tail -f`` command never finishes. What you\nneed is for ``tail`` to send its output to ``tr`` as it receives it. This is where\nthe :ref:`_piped ` special kwarg comes in handy:\n\n.. code-block:: python\n\n\tfor line in tr(_in=tail(\"-f\", \"test.log\", _piped=True), \"[:upper:]\", \"[:lower:]\", _iter=True):\n\t print(line)\n\t \nThis works by telling ``tail -f`` that it is being used in a pipeline, and that\nit should send its output line-by-line to ``tr``. By default, :ref:`piped` sends\nSTDOUT, but you can easily make it send STDERR instead by using ``_piped=\"err\"``\n" }, { "path": "docs/source/sections/redirection.rst", "content": ".. _redirection:\n\nRedirection\n===========\n\nsh can redirect the STDOUT and STDERR of a process to many different types of\ntargets, using the :ref:`_out ` and :ref:`_err ` special kwargs.\n\nFilename\n--------\n\nIf a string is used, it is assumed to be a filename. The filename is opened as\n\"wb\", meaning truncate-write and binary mode.\n\n.. code-block:: python\n\n import sh\n sh.ifconfig(_out=\"/tmp/interfaces\")\n\n.. seealso:: :ref:`faq_append`\n\nFile-like Object\n----------------\n\nYou may also use any object that supports ``.write(data)``, like\n:class:`io.StringIO`:\n\n.. code-block:: python\n\n import sh\n from io import StringIO\n\n buf = StringIO()\n sh.ifconfig(_out=buf)\n print(buf.getvalue())\n\n.. _red_func:\n\nFunction Callback\n-----------------\n\nA callback function may also be used as a target. The function must conform to\none of three signatures:\n\n.. py:function:: fn(data)\n :noindex:\n\n The function takes just the chunk of data from the process.\n\n.. py:function:: fn(data, stdin_queue)\n :noindex:\n\n In addition to the previous signature, the function also takes a\n :class:`queue.Queue`, which may be used to communicate programmatically with\n the process.\n\n.. py:function:: fn(data, stdin_queue, process)\n :noindex:\n\n In addition to the previous signature, the function takes a\n :class:`weakref.ref` to the :ref:`OProc ` object.\n\n.. seealso:: :ref:`callbacks`\n\n.. seealso:: :ref:`tutorial2`\n" }, { "path": "docs/source/sections/special_arguments.rst", "content": ".. _special_arguments:\n\n.. |def| replace:: Default value:\n\nSpecial Kwargs\n##############\n\nThese arguments alter a command's behavior. They are not passed to the program.\nYou can use them on any command that you run, but some may not be used together.\nsh will tell you if there are conflicts.\n\nTo set default special keyword arguments on *every* command run, you may use\n:ref:`default_arguments`.\n\nControlling Output\n==================\n\n.. _out:\n\n_out\n----\n|def| ``None``\n\nWhat to redirect STDOUT to. If this is a string, it will be treated as a file\nname. You may also pass a file object (or file-like object), an int\n(representing a file descriptor, like the result of :func:`os.pipe`), a\n:class:`io.StringIO` object, or a callable.\n\n.. code-block:: python\n\n import sh\n sh.ls(_out=\"/tmp/output\")\n\n.. seealso::\n :ref:`redirection`\n\t\t\n.. _err:\n\n_err\n----\n|def| ``None``\n\nWhat to redirect STDERR to. See :ref:`_out`.\n \n_err_to_out\n-----------\n|def| ``False``\n\nIf ``True``, duplicate the file descriptor bound to the process's STDOUT also to\nSTDERR, effectively causing STDERR and STDOUT to go to the same place.\n\n_encoding\n---------\n|def| ``sh.DEFAULT_ENCODING``\n\nThe character encoding of the process's STDOUT. By default, this is the\nlocale's default encoding.\n\t\t\t\n_decode_errors\n--------------\n.. versionadded:: 1.07.0\n\n|def| ``\"strict\"``\n\nThis is how Python should handle decoding errors of the process's output.\nBy default, this is ``\"strict\"``, but you can use any value that's valid\nto :meth:`bytes.decode`, such as ``\"ignore\"``.\n\t\t\n_tee\n----\n.. versionadded:: 1.07.0\n\n|def| ``None``\n\nAs of 1.07.0, any time redirection is used, either for STDOUT or STDERR, the\nrespective internal buffers are not filled. For example, if you're downloading\na file and using a callback on STDOUT, the internal STDOUT buffer, nor the pipe\nbuffer be filled with data from STDOUT. This option forces one of stderr\n(``_tee='err'``) or stdout (``_tee='out'`` or ``_tee=True``) to be filled\nanyways, in effect \"tee-ing\" the output into two places (the callback/redirect\nhandler, and the internal buffers).\n\n\n_truncate_exc\n-------------\n.. versionadded:: 1.12.0\n\n|def| ``True``\n\nWhether or not exception ouput should be truncated.\n\nExecution\n=========\n\n.. _fg:\n\n_fg\n---\n.. versionadded:: 1.12.0\n\n|def| ``False``\n\nRuns a command in the foreground, meaning it is spawned using :func:`os.spawnle()`. The current process's STDIN/OUT/ERR\nis :func:`os.dup2`'d to the new process and so the new process becomes the *foreground* of the shell executing the\nscript. This is only really useful when you want to launch a lean, interactive process that sh is having trouble\nrunning, for example, ssh.\n\n.. warning::\n\n ``_fg=True`` side-steps a lot of sh's functionality. You will not be returned a process object and most (likely\n all) other special kwargs will not work.\n\nIf you are looking for similar functionality, but still retaining sh's features, use the following:\n\n.. code-block:: python\n \n import sh\n import sys\n sh.your_command(_in=sys.stdin, _out=sys.stdout, _err=sys.stderr)\n\n\n.. _bg:\n\n_bg\n---\n|def| ``False``\n\nRuns a command in the background. The command will return immediately, and you\nwill have to run :meth:`RunningCommand.wait` on it to ensure it terminates.\n\n.. seealso:: :ref:`background`.\n\n.. _bg_exc:\n\n_bg_exc\n-------\n.. versionadded:: 1.12.9\n\n|def| ``True``\n\nAutomatically report exceptions for the background command. If you set this to\n``False`` you should make sure to call :meth:`RunningCommand.wait` or you may\nswallow exceptions that happen in the background command.\n\n.. _async_kw:\n\n_async\n------\n.. versionadded:: 2.0.0\n\n|def| ``False``\n\nAllows your command to become awaitable. Use in combination with :ref:`_iter `\nand ``async for`` to incrementally await output as it is produced.\n\n.. _env:\n\n_env\n----\n|def| ``None``\n\nA dictionary defining the only environment variables that will be made\naccessible to the process. If not specified, the calling process's environment\nvariables are used.\n\n.. note::\n\n This dictionary is the authoritative environment for the process. If you\n wish to change a single variable in your current environement, you must pass\n a copy of your current environment with the overriden variable to sh.\n\n.. seealso:: :ref:`environments`\n\n.. _timeout:\n\n_timeout\n--------\n|def| ``None``\n\nHow much time, in seconds, we should give the process to complete. If the\nprocess does not finish within the timeout, it will be sent the signal defined\nby :ref:`timeout_signal`.\n\n.. _timeout_signal:\n\n_timeout_signal\n---------------\n|def| ``signal.SIGKILL``\n\nThe signal to be sent to the process if :ref:`timeout` is not ``None``.\n\n_cwd\n----\n|def| ``None``\n\nA string that sets the current working directory of the process.\n\n.. _ok_code:\n\n_ok_code\n--------\n|def| ``0``\n\nEither an integer, a list, or a tuple containing the exit code(s) that are\nconsidered \"ok\", or in other words: do not raise an exception. Some misbehaved\nprograms use exit codes other than 0 to indicate success.\n\n.. code-block:: python\n\n import sh\n sh.weird_program(_ok_code=[0,3,5])\n\nIf the process is killed by a signal, a :ref:`signal_exc` is raised by\ndefault. This behavior could be blocked by appending a negative number to\n:ref:`ok_code` that represents the signal.\n\n.. code-block:: python\n\n import sh\n # the process won't raise SignalException if SIGINT, SIGKILL, or SIGTERM\n # are sent to kill the process\n p = sh.sleep(3, _bg=True, _ok_code=[0, -2, -9, -15])\n\n # No exception will be raised here\n p.kill()\n\n.. seealso:: :ref:`exit_codes`\n\n.. _new_session:\n\n_new_session\n------------\n|def| ``False``\n\nDetermines if our forked process will be executed in its own session via\n:func:`os.setsid`.\n\n.. versionchanged:: 2.0.0\n The default value of ``_new_session`` was changed from ``True`` to ``False``\n because it makes more sense for a launched process to default to being in\n the process group of python script, so that it receives SIGINTs correctly.\n\n.. note::\n\n If ``_new_session`` is ``False``, the forked process will be put into its\n own group via ``os.setpgrp()``. This way, the forked process, and all of\n it's children, are always alone in their own group that may be signalled\n directly, regardless of the value of ``_new_session``.\n\n.. seealso:: :ref:`architecture`\n\n.. _uid:\n\n_uid\n----\n.. versionadded:: 1.12.0\n\n|def| ``None``\n\nThe user id to assume before the child process calls :func:`os.execv`.\n\n_preexec_fn\n-----------\n.. versionadded:: 1.12.0\n\n|def| ``None``\n\nA function to be run directly before the child process calls :func:`os.execv`.\nTypically not used by normal users.\n\n.. _pass_fds:\n\n_pass_fds\n---------\n.. versionadded:: 1.13.0\n\n|def| ``{}`` (empty set)\n\nA whitelist iterable of integer file descriptors to be inherited by the child. Passing anything in this argument causes :ref:`_close_fds ` to be ``True``.\n\t\t\n.. _close_fds:\n\n_close_fds\n----------\n.. versionadded:: 1.13.0\n\n|def| ``True``\n\nCauses all inherited file descriptors besides stdin, stdout, and stderr to be automatically closed. This option is\nautomatically enabled when :ref:`_pass_fds ` is given a value.\n\nCommunication\n=============\n\n.. _in:\n\n_in\n---\n\n|def| ``None``\n\nSpecifies an argument for the process to use as its standard input. This may be\na string, a :class:`queue.Queue`, a file-like object, or any iterable.\n\n.. seealso:: :ref:`stdin`\n\t\t\n.. _piped:\n\n_piped\n------\n\n|def| ``None``\n\nMay be ``True``, ``\"out\"``, or ``\"err\"``. Signals a command that it is being\nused as the input to another command, so it should return its output\nincrementally as it receives it, instead of aggregating it all at once.\n\n.. seealso:: :ref:`Advanced Piping `\n\n.. _iter:\n\t\t\n_iter\n-----\n\n|def| ``None``\n\nMay be ``True``, ``\"out\"``, or ``\"err\"``. Puts a command in iterable mode. In\nthis mode, you can use a ``for`` or ``while`` loop to iterate over a command's\noutput in real-time.\n\n.. code-block:: python\n\n import sh \n for line in sh.cat(\"/tmp/file\", _iter=True):\n print(line)\n\n.. seealso:: :ref:`iterable`.\n\n.. _iter_noblock:\n\t\t\n_iter_noblock\n-------------\n|def| ``None``\n\nSame as :ref:`_iter `, except the loop will not block if there is no\noutput to iterate over. Instead, the output from the command will be\n:py:data:`errno.EWOULDBLOCK`.\n\n.. code-block:: python\n\n import sh\n import errno\n import time\n\n for line in sh.tail(\"-f\", \"stuff.log\", _iter_noblock=True):\n if line == errno.EWOULDBLOCK:\n print(\"doing something else...\")\n time.sleep(0.5)\n else:\n print(\"processing line!\")\n\n\n.. seealso:: :ref:`iterable`.\n\n.. _with:\n\n_with\n-----\n|def| ``False``\n\nExplicitly tells us that we're running a command in a ``with`` context. This is\nonly necessary if you're using a command in a ``with`` context **and** passing\nparameters to it.\n\n.. code-block:: python\n\n import sh\n with sh.contrib.sudo(password=\"abc123\", _with=True):\n print(sh.ls(\"/root\"))\n\n.. seealso:: :ref:`with_contexts`\n\n.. _done:\n\n_done\n-----\n.. versionadded:: 1.11.0\n\n|def| ``None``\n\nA callback that is *always* called when the command completes, even if it\ncompletes with an exit code that would raise an exception. After the callback\nis run, any exception that would be raised is raised.\n\nThe callback is passed the :ref:`RunningCommand ` instance, a\nboolean indicating success, and the exit code.\n\n.. include:: /examples/done.rst\n\t\t\nTTYs\n====\n\n.. _tty_in:\n\n_tty_in\n-------\n\n|def| ``False``, meaning a :func:`os.pipe` will be used.\n\nIf ``True``, sh creates a TTY for STDIN, essentially emulating a terminal, as if\nyour command was entered from the commandline. This is necessary for commands\nthat require STDIN to be a TTY.\n\n.. _tty_out:\n \n_tty_out\n--------\n\n|def| ``True``\n\nIf ``True``, sh creates a TTY for STDOUT, otherwise use a :func:`os.pipe`. This\nis necessary for commands that require STDOUT to be a TTY.\n\n.. seealso:: :ref:`faq_tty_out`\n\n.. _unify_ttys:\n\n_unify_ttys\n-----------\n.. versionadded:: 1.13.0\n\n|def| ``False``\n\nIf ``True``, sh will combine the STDOUT and STDIN TTY into a single\npseudo-terminal. This is sometimes required by picky programs which expect to be\ndealing with a single pseudo-terminal, like SSH.\n\n.. seealso:: :ref:`tutorial2`\n\n_tty_size\n---------\n\n|def| ``(20, 80)``\n\nThe (rows, columns) of stdout's TTY. Changing this may affect how much your\nprogram prints per line, for example.\n\nPerformance & Optimization\n==========================\n\n_in_bufsize\n-----------\n|def| ``0``\n\nThe STDIN buffer size. 0 for unbuffered, 1 for line buffered, anything else for\na buffer of that amount.\n\n.. _out_bufsize:\n\t\t\n_out_bufsize\n------------\n|def| ``1``\n\nThe STDOUT buffer size. 0 for unbuffered, 1 for line buffered, anything\nelse for a buffer of that amount.\n\n.. _err_bufsize:\n\n_err_bufsize\n------------\n|def| ``1``\n\nSame as :ref:`out_bufsize`, but with STDERR.\n\n.. _internal_bufsize:\n\t\t\n_internal_bufsize\n-----------------\n|def| ``3 * 1024**2`` chunks\n\nHow much of STDOUT/ERR your command will store internally. This value\nrepresents the *number of bufsize chunks* not the total number of bytes. For\nexample, if this value is 100, and STDOUT is line buffered, you will be able to\nretrieve 100 lines from STDOUT. If STDOUT is unbuffered, you will be able to\nretrieve only 100 characters.\n\t\t\n_no_out\n-------\n.. versionadded:: 1.07.0\n\n|def| ``False``\n\nDisables STDOUT being internally stored. This is useful for commands\nthat produce huge amounts of output that you don't need, that would\notherwise be hogging memory if stored internally by sh.\n\t\t\n_no_err\n-------\n.. versionadded:: 1.07.0\n\n|def| ``False``\n\nDisables STDERR being internally stored. This is useful for commands that\nproduce huge amounts of output that you don't need, that would otherwise be\nhogging memory if stored internally by sh.\n\t\t\n_no_pipe\n--------\n.. versionadded:: 1.07.0\n\n|def| ``False``\n\nSimilar to ``_no_out``, this explicitly tells the sh command that it will never\nbe used for piping its output into another command, so it should not fill its\ninternal pipe buffer with the process's output. This is also useful for\nconserving memory.\n\t\t\n\nProgram Arguments\n=================\n\nThese are options that affect how command options are fed into the program.\n\n_long_sep\n---------\n.. versionadded:: 1.12.0\n\n|def| ``\"=\"``\n\nThis is the character(s) that separate a program's long argument's key from the\nvalue, when using kwargs to specify your program's long arguments. For example,\nif your program expects a long argument in the form ``--name value``, the way to\nachieve this would be to set ``_long_sep=\" \"``.\n\n.. code-block:: python\n\n import sh\n sh.your_program(key=value, _long_sep=\" \")\n\nWould send the following list of arguments to your program:\n\n.. code-block:: python\n\n [\"--key value\"]\n\nIf your program expects the long argument name to be separate from its value,\npass ``None`` into ``_long_sep`` instead:\n\n.. code-block:: python\n\n import sh\n sh.your_program(key=value, _long_sep=None)\n\nWould send the following list of arguments to your program:\n\n.. code-block:: python\n\n [\"--key\", \"value\"]\n\n_long_prefix\n------------\n.. versionadded:: 1.12.0\n\n|def| ``\"--\"``\n\nThis is the character(s) that prefix a long argument for the program being run.\nSome programs use single dashes, for example, and do not understand double\ndashes.\n\n.. _preprocess:\n\n_arg_preprocess\n---------------\n.. versionadded:: 1.12.0\n\n|def| ``None``\n\nThis is an advanced option that allows you to rewrite a command's arguments on\nthe fly, based on other command arguments, or some other variable. It is really\nonly useful in conjunction with :ref:`baking `, and only currently used when\nconstructing :ref:`contrib ` wrappers.\n\nExample:\n\n.. code-block:: python\n\n import sh\n\n def processor(args, kwargs):\n return args, kwargs\n\n my_ls = sh.bake.ls(_arg_preprocess=processor)\n\n.. warning::\n\n The interface to the ``_arg_preprocess`` function may change without\n warning. It is generally only for internal sh use, so don't use it unless\n you absolutely have to.\n\nMisc\n====\n\n_log_msg\n--------\n\n|def| ``None``\n\n.. versionadded:: 1.12.0\n\nThis allows for a custom logging header for :ref:`command_class` instances. For example, the default logging looks like this:\n\n.. code-block:: python\n\n import logging\n import sh\n\n logging.basicConfig(level=logging.INFO)\n\n sh.ls(\"-l\")\n\n.. code-block:: none\n\n INFO:sh.command:: starting process\n INFO:sh.command:: process started\n INFO:sh.command:: process completed\n\nPeople can find this ``` to make it well-behaved. In\nparticular, the contrib version allows you to specify your password at execution\ntime via terminal input, or as a string in your script.\n\nTerminal Input\n^^^^^^^^^^^^^^\n\nVia a :ref:`with context `:\n\n.. code-block:: python\n\n import sh\n\n with sh.contrib.sudo:\n print(ls(\"/root\"))\n\nOr alternatively via :ref:`subcommands `:\n\n.. code-block:: python\n\n import sh\n print(sh.contrib.sudo.ls(\"/root\"))\n\nOutput:\n\n.. code-block:: none\n\n [sudo] password for youruser: *************\n your_root_files.txt\n\nIn the above example, ``sh.contrib.sudo`` automatically asks you for a password\nusing :func:`getpass.getpass` under the hood.\n\nThis method is the most secure, because it lowers the chances of doing something\ninsecure, like including your password in your python script, or by saying that\na particular user can execute anything inside of a particular script (the\nNOPASSWD method).\n\n.. note::\n\n ``sh.contrib.sudo`` does not do password caching like the sudo binary does.\n Thie means that each time a sudo command is run in your script, you will be\n asked to type in a password.\n\nString Input\n^^^^^^^^^^^^\n\nYou may also specify your password to ``sh.contrib.sudo`` as a string:\n\n.. code-block:: python\n\n import sh\n\n password = get_your_password()\n\n with sh.contrib.sudo(password=password, _with=True):\n print(ls(\"/root\"))\n\n.. warning::\n\n This method is less secure because it becomes tempting to hard-code your\n password into the python script, and that's a bad idea. However, it is more\n flexible, because it allows you to obtain your password from another source,\n so long as the end result is a string.\n\n/etc/sudoers NOPASSWD\n---------------------\n\nWith this method, you can use the raw ``sh.sudo`` command directly, because\nyou're being guaranteed that the system will not ask you for a password. It\nfirst requires you set up your user to have root execution privileges\n\nEdit your sudoers file:\n\n.. code-block:: none\n\n $> sudo visudo\n\nAdd or edit the line describing your user's permissions:\n\n.. code-block:: none\n\n yourusername ALL = (root) NOPASSWD: /path/to/your/program\n\nThis says ``yourusername`` on ``ALL`` hosts will be able to run as root, but\nonly root ``(root)`` (no other users), and that no password ``NOPASSWD`` will be\nasked of ``/path/to/your/program``.\n\n.. warning::\n \n This method can be insecure if an unprivileged user can edit your script,\n because the entire script will be exited as a privileged user. A malicious\n user could put something bad in this script.\n\n.. _sudo_raw:\n\nsh.sudo\n-------\n\nUsing the raw command ``sh.sudo`` (which resolves directly to the system's\n``sudo`` binary) without NOPASSWD is possible, provided you wire up the special\nkeyword arguments on your own to make it behave correctly. This method is\ndiscussed generally for educational purposes; if you take the time to wire up\n``sh.sudo`` on your own, then you have in essence just recreated\n:ref:`contrib_sudo`.\n\n.. code-block:: python\n\n import sh\n\n # password must end in a newline\n my_password = \"password\\n\"\n\n # -S says \"get the password from stdin\"\n my_sudo = sh.sudo.bake(\"-S\", _in=my_password)\n\n print(my_sudo.ls(\"root\"))\n\n_fg=True\n--------\n\nAnother less-obvious way of using sudo is by executing the raw ``sh.sudo``\ncommand but also putting it in the foreground. This way, sudo will work\ncorrectly automatically, by hooking up stdin/out/err automatically, and by\nasking you for a password if it requires one. The downsides of using\n:ref:`_fg=True `, however, are that you cannot capture its output -- everything is\njust printed to your terminal as if you ran it from a shell.\n\n.. code-block:: python\n\n import sh\n sh.sudo.ls(\"/root\", _fg=True)\n" }, { "path": "docs/source/sections/with.rst", "content": ".. _with_contexts:\n\n'With' Contexts\n===============\n\nCommands can be run within a Python ``with`` context. Popular commands using\nthis might be ``sudo`` or ``fakeroot``:\n\n.. code-block:: python\n\n\twith sh.contrib.sudo:\n\t print(ls(\"/root\"))\n\n.. seealso::\n\n :ref:`contrib_sudo`\n\t\t\nIf you need to run a command in a with context and pass in arguments, for\nexample, specifying a -p prompt with sudo, you need to use the :ref:`_with=True\n` This let's the command know that it's being run from a with context so\nit can behave correctly:\n\n.. code-block:: python\n\n\twith sh.contrib.sudo(k=True, _with=True):\n\t print(ls(\"/root\"))\n\t \n\n" }, { "path": "docs/source/tutorials/interacting_with_processes.rst", "content": ".. _tutorial2:\n\nEntering an SSH password\n========================\n\nHere we will attempt to SSH into a server and enter a password programmatically.\n\n.. note::\n\n It is recommended that you just ``ssh-copy-id`` to copy your public key to\n the server so you don't need to enter your password, but for the purposes of\n this demonstration, we try to enter a password.\n\nTo interact with a process, we need to assign a callback to STDOUT. The\ncallback signature we'll use will take a :class:`queue.Queue` object for the\nsecond argument, and we'll use that to send STDIN back to the process.\n\n.. seealso:: :ref:`red_func`\n\nHere's our first attempt:\n\n.. code-block:: python\n\n from sh import ssh\n\n def ssh_interact(line, stdin):\n line = line.strip()\n print(line)\n if line.endswith(\"password:\"):\n stdin.put(\"correcthorsebatterystaple\")\n\n ssh(\"10.10.10.100\", _out=ssh_interact)\n\nIf you run this (substituting an IP that you can SSH to), you'll notice that\nnothing is printed from within the callback. The problem has to do with STDOUT\nbuffering. By default, sh line-buffers STDOUT, which means that\n``ssh_interact`` will only receive output when sh encounters a newline in the\noutput. This is a problem because the password prompt has no newline:\n\n.. code-block:: none\n\n amoffat@10.10.10.100's password:\n\nBecause a newline is never encountered, nothing is sent to the ``ssh_interact``\ncallback. So we need to change the STDOUT buffering. We do this with the\n:ref:`_out_bufsize ` special kwarg. We'll set\nit to 0 for unbuffered output:\n\n.. code-block:: python\n\n from sh import ssh\n \n def ssh_interact(line, stdin):\n line = line.strip()\n print(line)\n if line.endswith(\"password:\"):\n stdin.put(\"correcthorsebatterystaple\")\n\n ssh(\"10.10.10.100\", _out=ssh_interact, _out_bufsize=0)\n\nIf you run this updated version, you'll notice a new problem. The output looks\nlike this:\n\n.. code-block:: none\n\n a\n m\n o\n f\n f\n a\n t\n @\n 1\n 0\n .\n 1\n 0\n .\n 1\n 0\n .\n 1\n 0\n 0\n '\n s\n \n p\n a\n s\n s\n w\n o\n r\n d\n :\n\nThis is because the chunks of STDOUT our callback is receiving are unbuffered,\nand are therefore individual characters, instead of entire lines. What we need\nto do now is aggregate this character-by-character data into something more\nmeaningful for us to test if the pattern ``password:`` has been sent, signifying\nthat SSH is ready for input.\n\nIt would make sense to encapsulate the variable we'll use for aggregating into\nsome kind of closure or class, but to keep it simple, we'll just use a global:\n\n.. code-block:: python\n\n from sh import ssh\n import sys\n\n aggregated = \"\"\n def ssh_interact(char, stdin):\n global aggregated\n sys.stdout.write(char.encode())\n sys.stdout.flush()\n aggregated += char\n if aggregated.endswith(\"password: \"):\n stdin.put(\"correcthorsebatterystaple\")\n\n ssh(\"10.10.10.100\", _out=ssh_interact, _out_bufsize=0)\n\nYou'll also notice that the example still doesn't work. There are two problems:\nThe first is that your password must end with a newline, as if you had typed it\nand hit the return key. This is because SSH has no idea how long your password\nis, and is line-buffering STDIN.\n\nThe second problem lies deeper in SSH. SSH needs a TTY attached to its STDIN in\norder to work properly. This tricks SSH into believing that it is interacting\nwith a real user in a real terminal session. To enable TTY, we can add the\n:ref:`_tty_in ` special kwarg. We also need to use :ref:`_unify_ttys ` special kwarg.\nThis tells sh to make STDOUT and STDIN come from a single pseudo-terminal, which is a requirement of SSH:\n\n.. code-block:: python\n\n from sh import ssh\n import sys\n\n aggregated = \"\"\n def ssh_interact(char, stdin):\n global aggregated\n sys.stdout.write(char.encode())\n sys.stdout.flush()\n aggregated += char\n if aggregated.endswith(\"password: \"):\n stdin.put(\"correcthorsebatterystaple\\n\")\n\n ssh(\"10.10.10.100\", _out=ssh_interact, _out_bufsize=0, _tty_in=True, _unify_ttys=True)\n \nAnd now our remote login script works!\n\n.. code-block:: none\n\n amoffat@10.10.10.100's password: \n Linux 10.10.10.100 testhost #1 SMP Tue Jun 21 10:29:24 EDT 2011 i686 GNU/Linux\n Ubuntu 10.04.2 LTS\n \n Welcome to Ubuntu!\n * Documentation: https://help.ubuntu.com/\n \n 66 packages can be updated.\n 53 updates are security updates.\n \n Ubuntu 10.04.2 LTS\n \n Welcome to Ubuntu!\n * Documentation: https://help.ubuntu.com/\n You have new mail.\n Last login: Thu Sep 13 03:53:00 2012 from some.ip.address\n amoffat@10.10.10.100:~$ \n\nSSH Contrib command\n-------------------\n\nThe above process can be simplified by using a :ref:`contrib`. The :ref:`SSH contrib command ` does\nall the ugly kwarg argument setup for you, and provides a simple but powerful interface for doing SSH password logins.\nPlease see the :ref:`SSH contrib command ` for more details about the exact api:\n\n.. code-block:: python\n\n from sh.contrib import ssh\n\n def ssh_interact(content, stdin):\n sys.stdout.write(content.cur_char)\n sys.stdout.flush()\n\n # automatically logs in with password and then presents subsequent content to\n # the ssh_interact callback\n ssh(\"10.10.10.100\", password=\"correcthorsebatterystaple\", interact=ssh_interact)\n\nHow you should REALLY be using SSH\n----------------------------------\n\nMany people want to learn how to enter an SSH password by script because they\nwant to execute remote commands on a server. Instead of trying to log in\nthrough SSH and then sending terminal input of the command to run, let's see how\nwe can do it another way.\n\nFirst, open a terminal and run ``ssh-copy-id yourservername``. You'll be asked\nto enter your password for the server. After entering your password, you'll be\nable to SSH into the server without needing a password again. This simplifies\nthings greatly for sh.\n\nThe second thing we want to do is use SSH's ability to pass a command to run\nto the server you're SSHing to. Here's how you can run ``ifconfig`` on a server\nwithout having to use that server's shell directly:\n\n.. code-block:: none\n\n ssh amoffat@10.10.10.100 ifconfig \n\nTranslating this to sh, it becomes:\n\n.. code-block:: python\n\n import sh\n\n print(sh.ssh(\"amoffat@10.10.10.100\", \"ifconfig\"))\n\nWe can make this even nicer by taking advantage of sh's :ref:`baking` to bind\nour server username/ip to a command object:\n\n.. code-block:: python\n\n import sh\n\n my_server = sh.ssh.bake(\"amoffat@10.10.10.100\")\n print(my_server(\"ifconfig\"))\n print(my_server(\"whoami\"))\n\nNow we have a reusable command object that we can use to call remote commands.\nBut there is room for one more improvement. We can also use sh's\n:ref:`subcommands` feature which expands attribute access into command\narguments:\n\n.. code-block:: python\n\n import sh\n\n my_server = sh.ssh.bake(\"amoffat@10.10.10.100\")\n print(my_server.ifconfig())\n print(my_server.whoami())\n" }, { "path": "docs/source/tutorials/real_time_output.rst", "content": ".. _tutorial1:\n\nTailing a real-time log file\n============================\n\nsh has the ability to respond to subprocesses in an event-driven fashion.\nA typical example of where this would be useful is tailing a log file for\na specific pattern, then responding to that value immediately::\n\n\tfrom sh import tail\n\t\n\tfor line in tail(\"-f\", \"info.log\", _iter=True):\n\t if \"ERROR\" in line:\n\t send_an_email_to_support(line)\n\t\t\t\n\t\t\t\nThe :ref:`_iter ` special kwarg takes a command that would normally block\nuntil completion, and turns its output into a real-time iterable.\n\n.. seealso:: :ref:`iterable`\n\nOf course, you can do more than just tail log files. Any program that\nproduces output can be iterated over. Say you wanted to send an email to a\ncoworker if their C code emits a warning:\n\n.. code-block:: python\n\n\tfrom sh import gcc, git\n\t\n\tfor line in gcc(\"-o\", \"awesome_binary\", \"awesome_source.c\", _iter=True):\n\t if \"warning\" in line:\n\t # parse out the relevant info\n\t filename, line, char, message = line.split(\":\", 3)\n\t \n\t # find the commit using git\n\t commit = git(\"blame\", \"-e\", filename, L=\"%d,%d\" % (line,line))\n\t \n\t # send them an email\n\t email_address = parse_email_from_commit_line(commit)\n\t send_email(email_address, message)\n\nUsing :ref:`_iter ` is a great way to respond to events from another\nprogram, but your blocks while you're looping, making you unable to do anything\nelse. To be truly event-driven, sh provides callbacks:\n\n.. code-block:: python\n\n\tfrom sh import tail\n\t\n\tdef process_log_line(line):\n\t if \"ERROR\" in line:\n\t send_an_email_to_support(line)\n\t\n\tprocess = tail(\"-f\", \"info.log\", _out=process_log_line, _bg=True)\n\t\n\t# ... do other stuff here ...\n\t\n\tprocess.wait()\n\t\nThe :ref:`_out ` special kwarg lets you to assign a callback to STDOUT.\nThis callback will receive each line of output from ``tail -f`` and allow you to\ndo the same processing that we did earlier.\n\n.. seealso:: :ref:`callbacks`\n\n.. seealso:: :ref:`redirection`\n" }, { "path": "docs/source/tutorials.rst", "content": "Tutorials\n=========\n\n.. toctree::\n tutorials/real_time_output\n tutorials/interacting_with_processes\n" }, { "path": "docs/source/usage.rst", "content": "Usage\n=====\n\n.. toctree::\n \n sections/passing_arguments\n sections/exit_codes\n sections/redirection\n sections/asynchronous_execution\n sections/baking\n sections/piping\n sections/subcommands\n sections/default_arguments\n sections/envs\n sections/stdin\n sections/with\n\n" }, { "path": "pyproject.toml", "content": "[tool.poetry]\nname = \"sh\"\nversion = \"2.2.2\"\ndescription = \"Python subprocess replacement\"\nauthors = [\"Andrew Moffat \"]\nreadme = \"README.rst\"\nmaintainers = [\n \"Andrew Moffat \",\n \"Erik Cederstrand \",\n]\nhomepage = \"https://sh.readthedocs.io/\"\nrepository = \"https://github.com/amoffat/sh\"\ndocumentation = \"https://sh.readthedocs.io/\"\nlicense = \"MIT\"\nclassifiers = [\n \"Development Status :: 5 - Production/Stable\",\n \"Environment :: Console\",\n \"Intended Audience :: Developers\",\n \"Intended Audience :: System Administrators\",\n \"License :: OSI Approved :: MIT License\",\n \"Programming Language :: Python\",\n \"Programming Language :: Python :: 3.8\",\n \"Programming Language :: Python :: 3.9\",\n \"Programming Language :: Python :: 3.10\",\n \"Programming Language :: Python :: 3.11\",\n \"Programming Language :: Python :: Implementation :: CPython\",\n \"Programming Language :: Python :: Implementation :: PyPy\",\n \"Topic :: Software Development :: Build Tools\",\n \"Topic :: Software Development :: Libraries :: Python Modules\",\n]\ninclude = [\n { path = \"CHANGELOG.md\", format = \"sdist\" },\n { path = \"MIGRATION.md\", format = \"sdist\" },\n { path = \"images\", format = \"sdist\" },\n { path = \"Makefile\", format = \"sdist\" },\n { path = \"tests\", format = \"sdist\" },\n { path = \"tox.ini\", format = \"sdist\" },\n { path = \"LICENSE.txt\", format = \"sdist\" },\n]\n\n[tool.poetry.dependencies]\npython = \">=3.8.1,<4.0\"\n\n[tool.poetry.group.dev.dependencies]\ntox = \"^4.6.4\"\nblack = \"^23.7.0\"\ncoverage = \"^7.2.7\"\nflake8 = \"^6.1.0\"\nrstcheck = \"^6.1.2\"\nsphinx = \">=1.6,<7\"\nsphinx-rtd-theme = \"^1.2.2\"\npytest = \"^7.4.0\"\nmypy = \"^1.4.1\"\ntoml = \"^0.10.2\"\n\n[build-system]\nrequires = [\"poetry-core>=1.0.0a5\"]\nbuild-backend = \"poetry.core.masonry.api\"\n" }, { "path": "sh.py", "content": "\"\"\"\nhttps://sh.readthedocs.io/en/latest/\nhttps://github.com/amoffat/sh\n\"\"\"\n\n# ===============================================================================\n# Copyright (C) 2011-2025 by Andrew Moffat\n#\n# Permission is hereby granted, free of charge, to any person obtaining a copy\n# of this software and associated documentation files (the \"Software\"), to deal\n# in the Software without restriction, including without limitation the rights\n# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n# copies of the Software, and to permit persons to whom the Software is\n# furnished to do so, subject to the following conditions:\n#\n# The above copyright notice and this permission notice shall be included in\n# all copies or substantial portions of the Software.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n# THE SOFTWARE.\n# ===============================================================================\nimport asyncio\nimport platform\nfrom collections import deque\nfrom collections.abc import Mapping\nfrom importlib import metadata\n\ntry:\n __version__ = metadata.version(\"sh\")\nexcept metadata.PackageNotFoundError: # pragma: no cover\n __version__ = \"unknown\"\n\nif \"windows\" in platform.system().lower(): # pragma: no cover\n raise ImportError(\n f\"sh {__version__} is currently only supported on Linux and macOS.\"\n )\n\nimport errno\nimport fcntl\nimport gc\nimport getpass\nimport glob as glob_module\nimport inspect\nimport logging\nimport os\nimport pty\nimport pwd\nimport re\nimport select\nimport signal\nimport stat\nimport struct\nimport sys\nimport termios\nimport textwrap\nimport threading\nimport time\nimport traceback\nimport tty\nimport warnings\nimport weakref\nfrom asyncio import Queue as AQueue\nfrom contextlib import contextmanager\nfrom functools import partial\nfrom io import BytesIO, StringIO, UnsupportedOperation\nfrom io import open as fdopen\nfrom locale import getpreferredencoding\nfrom queue import Empty, Queue\nfrom shlex import quote as shlex_quote\nfrom types import GeneratorType, ModuleType\nfrom typing import Any, Dict, Type, Union\n\n__project_url__ = \"https://github.com/amoffat/sh\"\n\nTEE_STDOUT = {True, \"out\", 1}\nTEE_STDERR = {\"err\", 2}\n\nDEFAULT_ENCODING = getpreferredencoding() or \"UTF-8\"\n\nIS_MACOS = platform.system() in (\"AIX\", \"Darwin\")\nSH_LOGGER_NAME = __name__\n\n# normally i would hate this idea of using a global to signify whether we are\n# running tests, because it breaks the assumption that what is running in the\n# tests is what will run live, but we ONLY use this in a place that has no\n# serious side-effects that could change anything. as long as we do that, it\n# should be ok\nRUNNING_TESTS = bool(int(os.environ.get(\"SH_TESTS_RUNNING\", \"0\")))\nFORCE_USE_SELECT = bool(int(os.environ.get(\"SH_TESTS_USE_SELECT\", \"0\")))\n\n# a re-entrant lock for pushd. this way, multiple threads that happen to use\n# pushd will all see the current working directory for the duration of the\n# with-context\nPUSHD_LOCK = threading.RLock()\n\n\ndef get_num_args(fn):\n return len(inspect.getfullargspec(fn).args)\n\n\n_unicode_methods = set(dir(\"\"))\n\nHAS_POLL = hasattr(select, \"poll\")\nPOLLER_EVENT_READ = 1\nPOLLER_EVENT_WRITE = 2\nPOLLER_EVENT_HUP = 4\nPOLLER_EVENT_ERROR = 8\n\n\nclass PollPoller:\n def __init__(self):\n self._poll = select.poll()\n # file descriptor <-> file object bidirectional maps\n self.fd_lookup = {}\n self.fo_lookup = {}\n\n def __nonzero__(self):\n return len(self.fd_lookup) != 0\n\n def __len__(self):\n return len(self.fd_lookup)\n\n def _set_fileobject(self, f):\n if hasattr(f, \"fileno\"):\n fd = f.fileno()\n self.fd_lookup[fd] = f\n self.fo_lookup[f] = fd\n else:\n self.fd_lookup[f] = f\n self.fo_lookup[f] = f\n\n def _remove_fileobject(self, f):\n if hasattr(f, \"fileno\"):\n fd = f.fileno()\n del self.fd_lookup[fd]\n del self.fo_lookup[f]\n else:\n del self.fd_lookup[f]\n del self.fo_lookup[f]\n\n def _get_file_descriptor(self, f):\n return self.fo_lookup.get(f)\n\n def _get_file_object(self, fd):\n return self.fd_lookup.get(fd)\n\n def _register(self, f, events):\n # f can be a file descriptor or file object\n self._set_fileobject(f)\n fd = self._get_file_descriptor(f)\n self._poll.register(fd, events)\n\n def register_read(self, f):\n self._register(f, select.POLLIN | select.POLLPRI)\n\n def register_write(self, f):\n self._register(f, select.POLLOUT)\n\n def register_error(self, f):\n self._register(f, select.POLLERR | select.POLLHUP | select.POLLNVAL)\n\n def unregister(self, f):\n fd = self._get_file_descriptor(f)\n self._poll.unregister(fd)\n self._remove_fileobject(f)\n\n def poll(self, timeout):\n if timeout is not None:\n # convert from seconds to milliseconds\n timeout *= 1000\n changes = self._poll.poll(timeout)\n results = []\n for fd, events in changes:\n f = self._get_file_object(fd)\n if events & (select.POLLIN | select.POLLPRI):\n results.append((f, POLLER_EVENT_READ))\n elif events & select.POLLOUT:\n results.append((f, POLLER_EVENT_WRITE))\n elif events & select.POLLHUP:\n results.append((f, POLLER_EVENT_HUP))\n elif events & (select.POLLERR | select.POLLNVAL):\n results.append((f, POLLER_EVENT_ERROR))\n return results\n\n\nclass SelectPoller:\n def __init__(self):\n self.rlist = []\n self.wlist = []\n self.xlist = []\n\n def __nonzero__(self):\n return len(self.rlist) + len(self.wlist) + len(self.xlist) != 0\n\n def __len__(self):\n return len(self.rlist) + len(self.wlist) + len(self.xlist)\n\n @staticmethod\n def _register(f, events):\n if f not in events:\n events.append(f)\n\n @staticmethod\n def _unregister(f, events):\n if f in events:\n events.remove(f)\n\n def register_read(self, f):\n self._register(f, self.rlist)\n\n def register_write(self, f):\n self._register(f, self.wlist)\n\n def register_error(self, f):\n self._register(f, self.xlist)\n\n def unregister(self, f):\n self._unregister(f, self.rlist)\n self._unregister(f, self.wlist)\n self._unregister(f, self.xlist)\n\n def poll(self, timeout):\n _in, _out, _err = select.select(self.rlist, self.wlist, self.xlist, timeout)\n results = []\n for f in _in:\n results.append((f, POLLER_EVENT_READ))\n for f in _out:\n results.append((f, POLLER_EVENT_WRITE))\n for f in _err:\n results.append((f, POLLER_EVENT_ERROR))\n return results\n\n\n# here we use an use a poller interface that transparently selects the most\n# capable poller (out of either select.select or select.poll). this was added\n# by zhangyafeikimi when he discovered that if the fds created internally by sh\n# numbered > 1024, select.select failed (a limitation of select.select). this\n# can happen if your script opens a lot of files\nPoller: Union[Type[SelectPoller], Type[PollPoller]] = SelectPoller\nif HAS_POLL and not FORCE_USE_SELECT:\n Poller = PollPoller\n\n\nclass ForkException(Exception):\n def __init__(self, orig_exc):\n msg = f\"\"\"\n\nOriginal exception:\n===================\n\n{textwrap.indent(orig_exc, \" \")}\n\"\"\"\n Exception.__init__(self, msg)\n\n\nclass ErrorReturnCodeMeta(type):\n \"\"\"a metaclass which provides the ability for an ErrorReturnCode (or\n derived) instance, imported from one sh module, to be considered the\n subclass of ErrorReturnCode from another module. this is mostly necessary\n in the tests, where we do assertRaises, but the ErrorReturnCode that the\n program we're testing throws may not be the same class that we pass to\n assertRaises\n \"\"\"\n\n def __subclasscheck__(self, o):\n other_bases = {b.__name__ for b in o.__bases__}\n return self.__name__ in other_bases or o.__name__ == self.__name__\n\n\nclass ErrorReturnCode(Exception):\n __metaclass__ = ErrorReturnCodeMeta\n\n \"\"\" base class for all exceptions as a result of a command's exit status\n being deemed an error. this base class is dynamically subclassed into\n derived classes with the format: ErrorReturnCode_NNN where NNN is the exit\n code number. the reason for this is it reduces boiler plate code when\n testing error return codes:\n\n try:\n some_cmd()\n except ErrorReturnCode_12:\n print(\"couldn't do X\")\n\n vs:\n try:\n some_cmd()\n except ErrorReturnCode as e:\n if e.exit_code == 12:\n print(\"couldn't do X\")\n\n it's not much of a savings, but i believe it makes the code easier to read \"\"\"\n\n truncate_cap = 750\n\n def __reduce__(self):\n return self.__class__, (self.full_cmd, self.stdout, self.stderr, self.truncate)\n\n def __init__(self, full_cmd, stdout, stderr, truncate=True):\n self.exit_code = self.exit_code # makes pylint happy\n self.full_cmd = full_cmd\n self.stdout = stdout\n self.stderr = stderr\n self.truncate = truncate\n\n exc_stdout = self.stdout\n if truncate:\n exc_stdout = exc_stdout[: self.truncate_cap]\n out_delta = len(self.stdout) - len(exc_stdout)\n if out_delta:\n exc_stdout += (f\"... ({out_delta} more, please see e.stdout)\").encode()\n\n exc_stderr = self.stderr\n if truncate:\n exc_stderr = exc_stderr[: self.truncate_cap]\n err_delta = len(self.stderr) - len(exc_stderr)\n if err_delta:\n exc_stderr += (f\"... ({err_delta} more, please see e.stderr)\").encode()\n\n msg = (\n f\"\\n\\n RAN: {self.full_cmd}\"\n f\"\\n\\n STDOUT:\\n{exc_stdout.decode(DEFAULT_ENCODING, 'replace')}\"\n f\"\\n\\n STDERR:\\n{exc_stderr.decode(DEFAULT_ENCODING, 'replace')}\"\n )\n\n super().__init__(msg)\n\n\nclass SignalException(ErrorReturnCode):\n pass\n\n\nclass TimeoutException(Exception):\n \"\"\"the exception thrown when a command is killed because a specified\n timeout (via _timeout or .wait(timeout)) was hit\"\"\"\n\n def __init__(self, exit_code, full_cmd):\n self.exit_code = exit_code\n self.full_cmd = full_cmd\n super(Exception, self).__init__()\n\n\nSIGNALS_THAT_SHOULD_THROW_EXCEPTION = {\n signal.SIGABRT,\n signal.SIGBUS,\n signal.SIGFPE,\n signal.SIGILL,\n signal.SIGINT,\n signal.SIGKILL,\n signal.SIGPIPE,\n signal.SIGQUIT,\n signal.SIGSEGV,\n signal.SIGTERM,\n signal.SIGSYS,\n}\n\n\n# we subclass AttributeError because:\n# https://github.com/ipython/ipython/issues/2577\n# https://github.com/amoffat/sh/issues/97#issuecomment-10610629\nclass CommandNotFound(AttributeError):\n pass\n\n\nrc_exc_regex = re.compile(r\"(ErrorReturnCode|SignalException)_((\\d+)|SIG[a-zA-Z]+)\")\nrc_exc_cache: Dict[str, Type[ErrorReturnCode]] = {}\n\nSIGNAL_MAPPING = {\n v: k for k, v in signal.__dict__.items() if re.match(r\"SIG[a-zA-Z]+\", k)\n}\n\n\ndef get_exc_from_name(name):\n \"\"\"takes an exception name, like:\n\n ErrorReturnCode_1\n SignalException_9\n SignalException_SIGHUP\n\n and returns the corresponding exception. this is primarily used for\n importing exceptions from sh into user code, for instance, to capture those\n exceptions\"\"\"\n\n exc = None\n try:\n return rc_exc_cache[name]\n except KeyError:\n m = rc_exc_regex.match(name)\n if m:\n base = m.group(1)\n rc_or_sig_name = m.group(2)\n\n if base == \"SignalException\":\n try:\n rc = -int(rc_or_sig_name)\n except ValueError:\n rc = -getattr(signal, rc_or_sig_name)\n else:\n rc = int(rc_or_sig_name)\n\n exc = get_rc_exc(rc)\n return exc\n\n\ndef get_rc_exc(rc):\n \"\"\"takes a exit code or negative signal number and produces an exception\n that corresponds to that return code. positive return codes yield\n ErrorReturnCode exception, negative return codes yield SignalException\n\n we also cache the generated exception so that only one signal of that type\n exists, preserving identity\"\"\"\n\n try:\n return rc_exc_cache[rc]\n except KeyError:\n pass\n\n if rc >= 0:\n name = f\"ErrorReturnCode_{rc}\"\n base = ErrorReturnCode\n else:\n name = f\"SignalException_{SIGNAL_MAPPING[abs(rc)]}\"\n base = SignalException\n\n exc = ErrorReturnCodeMeta(name, (base,), {\"exit_code\": rc})\n rc_exc_cache[rc] = exc\n return exc\n\n\n# we monkey patch glob. i'm normally generally against monkey patching, but i\n# decided to do this really un-intrusive patch because we need a way to detect\n# if a list that we pass into an sh command was generated from glob. the reason\n# being that glob returns an empty list if a pattern is not found, and so\n# commands will treat the empty list as no arguments, which can be a problem,\n# ie:\n#\n# ls(glob(\"*.ojfawe\"))\n#\n# ^ will show the contents of your home directory, because it's essentially\n# running ls([]) which, as a process, is just \"ls\".\n#\n# so we subclass list and monkey patch the glob function. nobody should be the\n# wiser, but we'll have results that we can make some determinations on\n_old_glob = glob_module.glob\n\n\nclass GlobResults(list):\n def __init__(self, path, results):\n self.path = path\n list.__init__(self, results)\n\n\ndef glob(path, *args, **kwargs):\n expanded = GlobResults(path, _old_glob(path, *args, **kwargs))\n return expanded\n\n\nglob_module.glob = glob # type: ignore\n\n\ndef canonicalize(path):\n return os.path.abspath(os.path.expanduser(path))\n\n\ndef _which(program, paths=None):\n \"\"\"takes a program name or full path, plus an optional collection of search\n paths, and returns the full path of the requested executable. if paths is\n specified, it is the entire list of search paths, and the PATH env is not\n used at all. otherwise, PATH env is used to look for the program\"\"\"\n\n def is_exe(file_path):\n return (\n os.path.exists(file_path)\n and os.access(file_path, os.X_OK)\n and os.path.isfile(os.path.realpath(file_path))\n )\n\n found_path = None\n fpath, fname = os.path.split(program)\n\n # if there's a path component, then we've specified a path to the program,\n # and we should just test if that program is executable. if it is, return\n if fpath:\n program = canonicalize(program)\n if is_exe(program):\n found_path = program\n\n # otherwise, we've just passed in the program name, and we need to search\n # the paths to find where it actually lives\n else:\n paths_to_search = []\n\n if isinstance(paths, (tuple, list)):\n paths_to_search.extend(paths)\n else:\n env_paths = os.environ.get(\"PATH\", \"\").split(os.pathsep)\n paths_to_search.extend(env_paths)\n\n for path in paths_to_search:\n exe_file = os.path.join(canonicalize(path), program)\n if is_exe(exe_file):\n found_path = exe_file\n break\n\n return found_path\n\n\ndef resolve_command_path(program):\n path = _which(program)\n if not path:\n # our actual command might have a dash in it, but we can't call\n # that from python (we have to use underscores), so we'll check\n # if a dash version of our underscore command exists and use that\n # if it does\n if \"_\" in program:\n path = _which(program.replace(\"_\", \"-\"))\n if not path:\n return None\n return path\n\n\ndef resolve_command(name, command_cls, baked_args=None):\n path = resolve_command_path(name)\n cmd = None\n if path:\n cmd = command_cls(path)\n if baked_args:\n cmd = cmd.bake(**baked_args)\n return cmd\n\n\nclass Logger:\n \"\"\"provides a memory-inexpensive logger. a gotcha about python's builtin\n logger is that logger objects are never garbage collected. if you create a\n thousand loggers with unique names, they'll sit there in memory until your\n script is done. with sh, it's easy to create loggers with unique names if\n we want our loggers to include our command arguments. for example, these\n are all unique loggers:\n\n ls -l\n ls -l /tmp\n ls /tmp\n\n so instead of creating unique loggers, and without sacrificing logging\n output, we use this class, which maintains as part of its state, the logging\n \"context\", which will be the very unique name. this allows us to get a\n logger with a very general name, eg: \"command\", and have a unique name\n appended to it via the context, eg: \"ls -l /tmp\" \"\"\"\n\n def __init__(self, name, context=None):\n self.name = name\n self.log = logging.getLogger(f\"{SH_LOGGER_NAME}.{name}\")\n self.context = self.sanitize_context(context)\n\n def _format_msg(self, msg, *a):\n if self.context:\n msg = f\"{self.context}: {msg}\"\n return msg % a\n\n @staticmethod\n def sanitize_context(context):\n if context:\n context = context.replace(\"%\", \"%%\")\n return context or \"\"\n\n def get_child(self, name, context):\n new_name = self.name + \".\" + name\n new_context = self.context + \".\" + context\n return Logger(new_name, new_context)\n\n def info(self, msg, *a):\n self.log.info(self._format_msg(msg, *a))\n\n def debug(self, msg, *a):\n self.log.debug(self._format_msg(msg, *a))\n\n def error(self, msg, *a):\n self.log.error(self._format_msg(msg, *a))\n\n def exception(self, msg, *a):\n self.log.exception(self._format_msg(msg, *a))\n\n\ndef default_logger_str(cmd, call_args, pid=None):\n if pid:\n s = f\"\"\n else:\n s = f\"\"\n return s\n\n\nclass RunningCommand:\n \"\"\"this represents an executing Command object. it is returned as the\n result of __call__() being executed on a Command instance. this creates a\n reference to a OProc instance, which is a low-level wrapper around the\n process that was exec'd\n\n this is the class that gets manipulated the most by user code, and so it\n implements various convenience methods and logical mechanisms for the\n underlying process. for example, if a user tries to access a\n backgrounded-process's stdout/err, the RunningCommand object is smart enough\n to know to wait() on the process to finish first. and when the process\n finishes, RunningCommand is smart enough to translate exit codes to\n exceptions.\"\"\"\n\n # these are attributes that we allow to pass through to OProc\n _OProc_attr_allowlist = {\n \"signal\",\n \"terminate\",\n \"kill\",\n \"kill_group\",\n \"signal_group\",\n \"pid\",\n \"sid\",\n \"pgid\",\n \"ctty\",\n \"input_thread_exc\",\n \"output_thread_exc\",\n \"bg_thread_exc\",\n }\n\n def __init__(self, cmd, call_args, stdin, stdout, stderr):\n # self.ran is used for auditing what actually ran. for example, in\n # exceptions, or if you just want to know what was ran after the\n # command ran\n self.ran = \" \".join([shlex_quote(str(arg)) for arg in cmd])\n\n self.call_args = call_args\n self.cmd = cmd\n\n self.process = None\n self._waited_until_completion = False\n should_wait = True\n spawn_process = True\n\n # if we're using an async for loop on this object, we need to put the underlying\n # iterable in no-block mode. however, we will only know if we're using an async\n # for loop after this object is constructed. so we'll set it to False now, but\n # then later set it to True if we need it\n self._force_noblock_iter = False\n\n # this event is used when we want to `await` a RunningCommand. see how it gets\n # used in self.__await__\n try:\n asyncio.get_running_loop()\n except RuntimeError:\n self.aio_output_complete = None\n else:\n self.aio_output_complete = asyncio.Event()\n\n # this is used to track if we've already raised StopIteration, and if we\n # have, raise it immediately again if the user tries to call next() on\n # us. https://github.com/amoffat/sh/issues/273\n self._stopped_iteration = False\n\n # with contexts shouldn't run at all yet, they prepend\n # to every command in the context\n if call_args[\"with\"]:\n spawn_process = False\n get_prepend_stack().append(self)\n\n if call_args[\"piped\"] or call_args[\"iter\"] or call_args[\"iter_noblock\"]:\n should_wait = False\n\n if call_args[\"async\"]:\n should_wait = False\n\n # we're running in the background, return self and let us lazily\n # evaluate\n if call_args[\"bg\"]:\n should_wait = False\n\n # redirection\n if call_args[\"err_to_out\"]:\n stderr = OProc.STDOUT\n\n done_callback = call_args[\"done\"]\n if done_callback:\n call_args[\"done\"] = partial(done_callback, self)\n\n # set up which stream should write to the pipe\n # TODO, make pipe None by default and limit the size of the Queue\n # in oproc.OProc\n pipe = OProc.STDOUT\n if call_args[\"iter\"] == \"out\" or call_args[\"iter\"] is True:\n pipe = OProc.STDOUT\n elif call_args[\"iter\"] == \"err\":\n pipe = OProc.STDERR\n\n if call_args[\"iter_noblock\"] == \"out\" or call_args[\"iter_noblock\"] is True:\n pipe = OProc.STDOUT\n elif call_args[\"iter_noblock\"] == \"err\":\n pipe = OProc.STDERR\n\n # there's currently only one case where we wouldn't spawn a child\n # process, and that's if we're using a with-context with our command\n self._spawned_and_waited = False\n if spawn_process:\n log_str_factory = call_args[\"log_msg\"] or default_logger_str\n logger_str = log_str_factory(self.ran, call_args)\n self.log = Logger(\"command\", logger_str)\n\n self.log.debug(\"starting process\")\n\n if should_wait:\n self._spawned_and_waited = True\n\n # this lock is needed because of a race condition where a background\n # thread, created in the OProc constructor, may try to access\n # self.process, but it has not been assigned yet\n process_assign_lock = threading.Lock()\n with process_assign_lock:\n self.process = OProc(\n self,\n self.log,\n cmd,\n stdin,\n stdout,\n stderr,\n self.call_args,\n pipe,\n process_assign_lock,\n )\n\n logger_str = log_str_factory(self.ran, call_args, self.process.pid)\n self.log.context = self.log.sanitize_context(logger_str)\n self.log.info(\"process started\")\n\n if should_wait:\n self.wait()\n\n def wait(self, timeout=None):\n \"\"\"waits for the running command to finish. this is called on all\n running commands, eventually, except for ones that run in the background\n\n if timeout is a number, it is the number of seconds to wait for the process to\n resolve. otherwise block on wait.\n\n this function can raise a TimeoutException, either because of a `_timeout` on\n the command itself as it was\n launched, or because of a timeout passed into this method.\n \"\"\"\n if not self._waited_until_completion:\n # if we've been given a timeout, we need to poll is_alive()\n if timeout is not None:\n waited_for = 0\n sleep_amt = 0.1\n alive = False\n exit_code = None\n if timeout < 0:\n raise RuntimeError(\"timeout cannot be negative\")\n\n # while we still have time to wait, run this loop\n # notice that alive and exit_code are only defined in this loop, but\n # the loop is also guaranteed to run, defining them, given the\n # constraints that timeout is non-negative\n while waited_for <= timeout:\n alive, exit_code = self.process.is_alive()\n\n # if we're alive, we need to wait some more, but let's sleep\n # before we poll again\n if alive:\n time.sleep(sleep_amt)\n waited_for += sleep_amt\n\n # but if we're not alive, we're done waiting\n else:\n break\n\n # if we've made it this far, and we're still alive, then it means we\n # timed out waiting\n if alive:\n raise TimeoutException(None, self.ran)\n\n # if we didn't time out, we fall through and let the rest of the code\n # handle exit_code. notice that we set _waited_until_completion here,\n # only if we didn't time out. this allows us to re-wait again on\n # timeout, if we catch the TimeoutException in the parent frame\n self._waited_until_completion = True\n\n else:\n exit_code = self.process.wait()\n self._waited_until_completion = True\n\n if self.process.timed_out:\n # if we timed out, our exit code represents a signal, which is\n # negative, so let's make it positive to store in our\n # TimeoutException\n raise TimeoutException(-exit_code, self.ran)\n\n else:\n self.handle_command_exit_code(exit_code)\n\n # if an iterable command is using an instance of OProc for its stdin,\n # wait on it. the process is probably set to \"piped\", which means it\n # won't be waited on, which means exceptions won't propagate up to the\n # main thread. this allows them to bubble up\n if self.process._stdin_process:\n self.process._stdin_process.command.wait()\n\n self.log.debug(\"process completed\")\n return self\n\n def is_alive(self):\n \"\"\"returns whether or not we're still alive. this call has side-effects on\n OProc\"\"\"\n return self.process.is_alive()[0]\n\n def handle_command_exit_code(self, code):\n \"\"\"here we determine if we had an exception, or an error code that we\n weren't expecting to see. if we did, we create and raise an exception\n \"\"\"\n ca = self.call_args\n exc_class = get_exc_exit_code_would_raise(code, ca[\"ok_code\"], ca[\"piped\"])\n if exc_class:\n exc = exc_class(\n self.ran, self.process.stdout, self.process.stderr, ca[\"truncate_exc\"]\n )\n raise exc\n\n @property\n def stdout(self):\n self.wait()\n return self.process.stdout\n\n @property\n def stderr(self):\n self.wait()\n return self.process.stderr\n\n @property\n def exit_code(self):\n self.wait()\n return self.process.exit_code\n\n def __len__(self):\n return len(str(self))\n\n def __enter__(self):\n \"\"\"we don't actually do anything here because anything that should have\n been done would have been done in the Command.__call__ call.\n essentially all that has to happen is the command be pushed on the\n prepend stack.\"\"\"\n pass\n\n def __iter__(self):\n return self\n\n def __next__(self):\n \"\"\"allow us to iterate over the output of our command\"\"\"\n\n if self._stopped_iteration:\n raise StopIteration()\n\n pq = self.process._pipe_queue\n\n # the idea with this is, if we're using regular `_iter` (non-asyncio), then we\n # want to have blocking be True when we read from the pipe queue, so our cpu\n # doesn't spin too fast. however, if we *are* using asyncio (an async for loop),\n # then we want non-blocking pipe queue reads, because we'll do an asyncio.sleep,\n # in the coroutine that is doing the iteration, this way coroutines have better\n # yielding (see queue_connector in __aiter__).\n block_pq_read = not self._force_noblock_iter\n\n # we do this because if get blocks, we can't catch a KeyboardInterrupt\n # so the slight timeout allows for that.\n while True:\n try:\n chunk = pq.get(block_pq_read, self.call_args[\"iter_poll_time\"])\n except Empty:\n if self.call_args[\"iter_noblock\"] or self._force_noblock_iter:\n return errno.EWOULDBLOCK\n else:\n if chunk is None:\n self.wait()\n self._stopped_iteration = True\n raise StopIteration()\n try:\n return chunk.decode(\n self.call_args[\"encoding\"], self.call_args[\"decode_errors\"]\n )\n except UnicodeDecodeError:\n return chunk\n\n def __await__(self):\n async def wait_for_completion():\n await self.aio_output_complete.wait()\n if self.call_args[\"return_cmd\"]:\n # We know the command has completed already,\n # but need to catch exceptions\n self.wait()\n return self\n else:\n return str(self)\n\n return wait_for_completion().__await__()\n\n def __aiter__(self):\n # maxsize is critical to making sure our queue_connector function below yields\n # when it awaits _aio_queue.put(chunk). if we didn't have a maxsize, our loop\n # would happily iterate through `chunk in self` and put onto the queue without\n # any blocking, and therefore no yielding, which would prevent other coroutines\n # from running.\n self._aio_queue = AQueue(maxsize=1)\n self._force_noblock_iter = True\n\n # the sole purpose of this coroutine is to connect our pipe_queue (which is\n # being populated by a thread) to an asyncio-friendly queue. then, in __anext__,\n # we can iterate over that asyncio queue.\n async def queue_connector():\n try:\n # this will spin as fast as possible if there's no data to read,\n # thanks to self._force_noblock_iter. so we sleep below.\n for chunk in self:\n if chunk == errno.EWOULDBLOCK:\n # let us have better coroutine yielding.\n await asyncio.sleep(0.01)\n else:\n await self._aio_queue.put(chunk)\n finally:\n await self._aio_queue.put(None)\n\n task = asyncio.create_task(queue_connector())\n self._aio_task = task\n return self\n\n async def __anext__(self):\n chunk = await self._aio_queue.get()\n if chunk is not None:\n return chunk\n else:\n exc = self._aio_task.exception()\n if exc is not None:\n raise exc\n raise StopAsyncIteration\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n if self.call_args[\"with\"] and get_prepend_stack():\n get_prepend_stack().pop()\n\n def __str__(self):\n if self.process and self.stdout:\n return self.stdout.decode(\n self.call_args[\"encoding\"], self.call_args[\"decode_errors\"]\n )\n return \"\"\n\n def __eq__(self, other):\n return id(self) == id(other)\n\n def __contains__(self, item):\n return item in str(self)\n\n def __getattr__(self, p):\n # let these three attributes pass through to the OProc object\n if p in self._OProc_attr_allowlist:\n if self.process:\n return getattr(self.process, p)\n else:\n raise AttributeError\n\n # see if strings have what we're looking for\n if p in _unicode_methods:\n return getattr(str(self), p)\n\n raise AttributeError\n\n def __repr__(self):\n try:\n return str(self)\n except UnicodeDecodeError:\n if self.process:\n if self.stdout:\n return repr(self.stdout)\n return repr(\"\")\n\n def __long__(self):\n return int(str(self).strip())\n\n def __float__(self):\n return float(str(self).strip())\n\n def __int__(self):\n return int(str(self).strip())\n\n\ndef output_redirect_is_filename(out):\n return isinstance(out, str) or hasattr(out, \"__fspath__\")\n\n\ndef get_prepend_stack():\n tl = Command.thread_local\n if not hasattr(tl, \"_prepend_stack\"):\n tl._prepend_stack = []\n return tl._prepend_stack\n\n\ndef special_kwarg_validator(passed_kwargs, merged_kwargs, invalid_list):\n s1 = set(passed_kwargs.keys())\n invalid_args = []\n\n for elem in invalid_list:\n if callable(elem):\n fn = elem\n ret = fn(passed_kwargs, merged_kwargs)\n invalid_args.extend(ret)\n\n else:\n elem, error_msg = elem\n\n if s1.issuperset(elem):\n invalid_args.append((elem, error_msg))\n\n return invalid_args\n\n\ndef get_fileno(ob):\n # in py2, this will return None. in py3, it will return an method that\n # raises when called\n fileno_meth = getattr(ob, \"fileno\", None)\n\n fileno = None\n if fileno_meth:\n # py3 StringIO objects will report a fileno, but calling it will raise\n # an exception\n try:\n fileno = fileno_meth()\n except UnsupportedOperation:\n pass\n elif isinstance(ob, (int,)) and ob >= 0:\n fileno = ob\n\n return fileno\n\n\ndef ob_is_fd_based(ob):\n return get_fileno(ob) is not None\n\n\ndef ob_is_tty(ob):\n \"\"\"checks if an object (like a file-like object) is a tty.\"\"\"\n fileno = get_fileno(ob)\n is_tty = False\n if fileno is not None:\n is_tty = os.isatty(fileno)\n return is_tty\n\n\ndef ob_is_pipe(ob):\n fileno = get_fileno(ob)\n is_pipe = False\n if fileno:\n fd_stat = os.fstat(fileno)\n is_pipe = stat.S_ISFIFO(fd_stat.st_mode)\n return is_pipe\n\n\ndef output_iterator_validator(passed_kwargs, merged_kwargs):\n invalid = []\n if passed_kwargs.get(\"no_out\") and passed_kwargs.get(\"iter\") in (True, \"out\"):\n error = \"You cannot iterate over output if there is no output\"\n invalid.append(((\"no_out\", \"iter\"), error))\n return invalid\n\n\ndef tty_in_validator(passed_kwargs, merged_kwargs):\n # here we'll validate that people aren't randomly shotgun-debugging different tty\n # options and hoping that they'll work, without understanding what they do\n pairs = ((\"tty_in\", \"in\"), (\"tty_out\", \"out\"))\n invalid = []\n for tty_type, std in pairs:\n if tty_type in passed_kwargs and ob_is_tty(passed_kwargs.get(std, None)):\n error = (\n f\"`_{std}` is a TTY already, so so it doesn't make sense to set up a\"\n f\" TTY with `_{tty_type}`\"\n )\n invalid.append(((tty_type, std), error))\n\n # if unify_ttys is set, then both tty_in and tty_out must both be True\n if merged_kwargs[\"unify_ttys\"] and not (\n merged_kwargs[\"tty_in\"] and merged_kwargs[\"tty_out\"]\n ):\n invalid.append(\n (\n (\"unify_ttys\", \"tty_in\", \"tty_out\"),\n \"`_tty_in` and `_tty_out` must both be True if `_unify_ttys` is True\",\n )\n )\n\n return invalid\n\n\ndef fg_validator(passed_kwargs, merged_kwargs):\n \"\"\"fg is not valid with basically every other option\"\"\"\n\n invalid = []\n msg = \"\"\"\\\n_fg is invalid with nearly every other option, see warning and workaround here:\n\n https://sh.readthedocs.io/en/latest/sections/special_arguments.html#fg\"\"\"\n allowlist = {\"env\", \"fg\", \"cwd\", \"ok_code\"}\n offending = set(passed_kwargs.keys()) - allowlist\n\n if \"fg\" in passed_kwargs and passed_kwargs[\"fg\"] and offending:\n invalid.append((\"fg\", msg))\n return invalid\n\n\ndef bufsize_validator(passed_kwargs, merged_kwargs):\n \"\"\"a validator to prevent a user from saying that they want custom\n buffering when they're using an in/out object that will be os.dup'ed to the\n process, and has its own buffering. an example is a pipe or a tty. it\n doesn't make sense to tell them to have a custom buffering, since the os\n controls this.\"\"\"\n invalid = []\n\n in_ob = passed_kwargs.get(\"in\", None)\n out_ob = passed_kwargs.get(\"out\", None)\n\n in_buf = passed_kwargs.get(\"in_bufsize\", None)\n out_buf = passed_kwargs.get(\"out_bufsize\", None)\n\n in_no_buf = ob_is_fd_based(in_ob)\n out_no_buf = ob_is_fd_based(out_ob)\n\n err = \"Can't specify an {target} bufsize if the {target} target is a pipe or TTY\"\n\n if in_no_buf and in_buf is not None:\n invalid.append(((\"in\", \"in_bufsize\"), err.format(target=\"in\")))\n\n if out_no_buf and out_buf is not None:\n invalid.append(((\"out\", \"out_bufsize\"), err.format(target=\"out\")))\n\n return invalid\n\n\ndef env_validator(passed_kwargs, merged_kwargs):\n \"\"\"a validator to check that env is a dictionary and that all environment variable\n keys and values are strings. Otherwise, we would exit with a confusing exit code\n 255.\"\"\"\n invalid = []\n\n env = passed_kwargs.get(\"env\", None)\n if env is None:\n return invalid\n\n if not isinstance(env, Mapping):\n invalid.append((\"env\", f\"env must be dict-like. Got {env!r}\"))\n return invalid\n\n for k, v in passed_kwargs[\"env\"].items():\n if not isinstance(k, str):\n invalid.append((\"env\", f\"env key {k!r} must be a str\"))\n if not isinstance(v, str):\n invalid.append((\"env\", f\"value {v!r} of env key {k!r} must be a str\"))\n\n return invalid\n\n\nclass Command:\n \"\"\"represents an un-run system program, like \"ls\" or \"cd\". because it\n represents the program itself (and not a running instance of it), it should\n hold very little state. in fact, the only state it does hold is baked\n arguments.\n\n when a Command object is called, the result that is returned is a\n RunningCommand object, which represents the Command put into an execution\n state.\"\"\"\n\n thread_local = threading.local()\n RunningCommandCls = RunningCommand\n\n _call_args: Dict[str, Any] = {\n \"fg\": False, # run command in foreground\n # run a command in the background. commands run in the background\n # ignore SIGHUP and do not automatically exit when the parent process\n # ends\n \"bg\": False,\n # automatically report exceptions for background commands\n \"bg_exc\": True,\n \"with\": False, # prepend the command to every command after it\n \"in\": None,\n \"out\": None, # redirect STDOUT\n \"err\": None, # redirect STDERR\n \"err_to_out\": None, # redirect STDERR to STDOUT\n # stdin buffer size\n # 1 for line, 0 for unbuffered, any other number for that amount\n \"in_bufsize\": 0,\n # stdout buffer size, same values as above\n \"out_bufsize\": 1,\n \"err_bufsize\": 1,\n # this is how big the output buffers will be for stdout and stderr.\n # this is essentially how much output they will store from the process.\n # we use a deque, so if it overflows past this amount, the first items\n # get pushed off as each new item gets added.\n #\n # NOTICE\n # this is not a *BYTE* size, this is a *CHUNK* size...meaning, that if\n # you're buffering out/err at 1024 bytes, the internal buffer size will\n # be \"internal_bufsize\" CHUNKS of 1024 bytes\n \"internal_bufsize\": 3 * 1024**2,\n \"env\": None,\n \"piped\": None,\n \"iter\": None,\n \"iter_noblock\": None,\n # the amount of time to sleep between polling for the iter output queue\n \"iter_poll_time\": 0.1,\n \"ok_code\": 0,\n \"cwd\": None,\n # the separator delimiting between a long-argument's name and its value\n # setting this to None will cause name and value to be two separate\n # arguments, like for short options\n # for example, --arg=derp, '=' is the long_sep\n \"long_sep\": \"=\",\n # the prefix used for long arguments\n \"long_prefix\": \"--\",\n # this is for programs that expect their input to be from a terminal.\n # ssh is one of those programs\n \"tty_in\": False,\n \"tty_out\": True,\n \"unify_ttys\": False,\n \"encoding\": DEFAULT_ENCODING,\n \"decode_errors\": \"strict\",\n # how long the process should run before it is auto-killed\n \"timeout\": None,\n \"timeout_signal\": signal.SIGKILL,\n # TODO write some docs on \"long-running processes\"\n # these control whether or not stdout/err will get aggregated together\n # as the process runs. this has memory usage implications, so sometimes\n # with long-running processes with a lot of data, it makes sense to\n # set these to true\n \"no_out\": False,\n \"no_err\": False,\n \"no_pipe\": False,\n # if any redirection is used for stdout or stderr, internal buffering\n # of that data is not stored. this forces it to be stored, as if\n # the output is being T'd to both the redirected destination and our\n # internal buffers\n \"tee\": None,\n # will be called when a process terminates regardless of exception\n \"done\": None,\n # a tuple (rows, columns) of the desired size of both the stdout and\n # stdin ttys, if ttys are being used\n \"tty_size\": (24, 80),\n # whether or not our exceptions should be truncated\n \"truncate_exc\": True,\n # a function to call after the child forks but before the process execs\n \"preexec_fn\": None,\n # UID to set after forking. Requires root privileges. Not supported on\n # Windows.\n \"uid\": None,\n # put the forked process in its own process session?\n \"new_session\": False,\n # put the forked process in its own process group?\n \"new_group\": False,\n # pre-process args passed into __call__. only really useful when used\n # in .bake()\n \"arg_preprocess\": None,\n # a callable that produces a log message from an argument tuple of the\n # command and the args\n \"log_msg\": None,\n # whether or not to close all inherited fds. typically, this should be True,\n # as inheriting fds can be a security vulnerability\n \"close_fds\": True,\n # a allowlist of the integer fds to pass through to the child process. setting\n # this forces close_fds to be True\n \"pass_fds\": set(),\n # return an instance of RunningCommand always. if this isn't True, then\n # sometimes we may return just a plain unicode string\n \"return_cmd\": False,\n \"async\": False,\n }\n\n # this is a collection of validators to make sure the special kwargs make\n # sense\n _kwarg_validators = (\n ((\"err\", \"err_to_out\"), \"Stderr is already being redirected\"),\n ((\"piped\", \"iter\"), \"You cannot iterate when this command is being piped\"),\n (\n (\"piped\", \"no_pipe\"),\n \"Using a pipe doesn't make sense if you've disabled the pipe\",\n ),\n output_iterator_validator,\n ((\"close_fds\", \"pass_fds\"), \"Passing `pass_fds` forces `close_fds` to be True\"),\n tty_in_validator,\n bufsize_validator,\n env_validator,\n fg_validator,\n )\n\n def __init__(self, path, search_paths=None):\n found = _which(path, search_paths)\n\n self._path = \"\"\n\n # is the command baked (aka, partially applied)?\n self._partial = False\n self._partial_baked_args = []\n self._partial_call_args = {}\n\n # bugfix for functools.wraps. issue #121\n self.__name__ = str(self)\n\n if not found:\n raise CommandNotFound(path)\n\n # the reason why we set the values early in the constructor, and again\n # here, is for people who have tools that inspect the stack on\n # exception. if CommandNotFound is raised, we need self._path and the\n # other attributes to be set correctly, so repr() works when they're\n # inspecting the stack. issue #304\n self._path = found\n self.__name__ = str(self)\n\n def __getattribute__(self, name):\n # convenience\n get_attr = partial(object.__getattribute__, self)\n val = None\n\n if name.startswith(\"_\"):\n val = get_attr(name)\n\n elif name == \"bake\":\n val = get_attr(\"bake\")\n\n # here we have a way of getting past shadowed subcommands. for example,\n # if \"git bake\" was a thing, we wouldn't be able to do `git.bake()`\n # because `.bake()` is already a method. so we allow `git.bake_()`\n elif name.endswith(\"_\"):\n name = name[:-1]\n\n if val is None:\n val = get_attr(\"bake\")(name)\n\n return val\n\n @classmethod\n def _extract_call_args(cls, kwargs):\n \"\"\"takes kwargs that were passed to a command's __call__ and extracts\n out the special keyword arguments, we return a tuple of special keyword\n args, and kwargs that will go to the exec'ed command\"\"\"\n\n kwargs = kwargs.copy()\n call_args = {}\n for parg, default in cls._call_args.items():\n key = \"_\" + parg\n\n if key in kwargs:\n call_args[parg] = kwargs[key]\n del kwargs[key]\n\n merged_args = cls._call_args.copy()\n merged_args.update(call_args)\n invalid_kwargs = special_kwarg_validator(\n call_args, merged_args, cls._kwarg_validators\n )\n\n if invalid_kwargs:\n exc_msg = []\n for kwarg, error_msg in invalid_kwargs:\n exc_msg.append(f\" {kwarg!r}: {error_msg}\")\n exc_msg = \"\\n\".join(exc_msg)\n raise TypeError(f\"Invalid special arguments:\\n\\n{exc_msg}\\n\")\n\n return call_args, kwargs\n\n def bake(self, *args, **kwargs):\n \"\"\"returns a new Command object after baking(freezing) the given\n command arguments which are used automatically when its exec'ed\n\n special keyword arguments can be temporary baked and additionally be\n overridden in __call__ or in subsequent bakes (basically setting\n defaults)\"\"\"\n\n # construct the base Command\n fn = type(self)(self._path)\n fn._partial = True\n\n call_args, kwargs = self._extract_call_args(kwargs)\n\n fn._partial_call_args.update(self._partial_call_args)\n fn._partial_call_args.update(call_args)\n fn._partial_baked_args.extend(self._partial_baked_args)\n sep = call_args.get(\"long_sep\", self._call_args[\"long_sep\"])\n prefix = call_args.get(\"long_prefix\", self._call_args[\"long_prefix\"])\n fn._partial_baked_args.extend(compile_args(args, kwargs, sep, prefix))\n return fn\n\n def __str__(self):\n if not self._partial_baked_args:\n return self._path\n baked_args = \" \".join(shlex_quote(arg) for arg in self._partial_baked_args)\n return f\"{self._path} {baked_args}\"\n\n def __eq__(self, other):\n return str(self) == str(other)\n\n def __repr__(self):\n return f\"\"\n\n def __enter__(self):\n self(_with=True)\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n get_prepend_stack().pop()\n\n def __call__(self, *args, **kwargs):\n kwargs = kwargs.copy()\n args = list(args)\n\n # this will hold our final command, including arguments, that will be\n # exec'ed\n cmd = []\n\n # this will hold a complete mapping of all our special keyword arguments\n # and their values\n call_args = self.__class__._call_args.copy()\n\n # aggregate any 'with' contexts\n for prepend in get_prepend_stack():\n pcall_args = prepend.call_args.copy()\n # don't pass the 'with' call arg\n pcall_args.pop(\"with\", None)\n\n call_args.update(pcall_args)\n # we do not prepend commands used as a 'with' context as they will\n # be prepended to any nested commands\n if not kwargs.get(\"_with\", False):\n cmd.extend(prepend.cmd)\n\n cmd.append(self._path)\n\n # do we have an argument pre-processor? if so, run it. we need to do\n # this early, so that args, kwargs are accurate\n preprocessor = self._partial_call_args.get(\"arg_preprocess\", None)\n if preprocessor:\n args, kwargs = preprocessor(args, kwargs)\n\n # here we extract the special kwargs and override any\n # special kwargs from the possibly baked command\n extracted_call_args, kwargs = self._extract_call_args(kwargs)\n\n call_args.update(self._partial_call_args)\n call_args.update(extracted_call_args)\n\n # handle a None. this is added back only to not break the api in the\n # 1.* version. TODO remove this in 2.0, as \"ok_code\", if specified,\n # should always be a definitive value or list of values, and None is\n # ambiguous\n if call_args[\"ok_code\"] is None:\n call_args[\"ok_code\"] = 0\n\n if not getattr(call_args[\"ok_code\"], \"__iter__\", None):\n call_args[\"ok_code\"] = [call_args[\"ok_code\"]]\n\n # determine what our real STDIN is. is it something explicitly passed into\n # _in?\n stdin = call_args[\"in\"]\n\n # now that we have our stdin, let's figure out how we should handle it\n if isinstance(stdin, RunningCommand):\n if stdin.call_args[\"piped\"]:\n stdin = stdin.process\n else:\n stdin = stdin.process._pipe_queue\n\n processed_args = compile_args(\n args, kwargs, call_args[\"long_sep\"], call_args[\"long_prefix\"]\n )\n\n # makes sure our arguments are broken up correctly\n split_args = self._partial_baked_args + processed_args\n\n final_args = split_args\n\n cmd.extend(final_args)\n\n # if we're running in foreground mode, we need to completely bypass\n # launching a RunningCommand and OProc and just do a spawn\n if call_args[\"fg\"]:\n cwd = call_args[\"cwd\"] or os.getcwd()\n with pushd(cwd):\n if call_args[\"env\"] is None:\n exit_code = os.spawnv(os.P_WAIT, cmd[0], cmd)\n else:\n exit_code = os.spawnve(os.P_WAIT, cmd[0], cmd, call_args[\"env\"])\n\n exc_class = get_exc_exit_code_would_raise(\n exit_code, call_args[\"ok_code\"], call_args[\"piped\"]\n )\n if exc_class:\n ran = \" \".join(cmd)\n exc = exc_class(ran, b\"\", b\"\", call_args[\"truncate_exc\"])\n raise exc\n return None\n\n # stdout redirection\n stdout = call_args[\"out\"]\n if output_redirect_is_filename(stdout):\n stdout = open(str(stdout), \"wb\")\n\n # stderr redirection\n stderr = call_args[\"err\"]\n if output_redirect_is_filename(stderr):\n stderr = open(str(stderr), \"wb\")\n\n rc = self.__class__.RunningCommandCls(cmd, call_args, stdin, stdout, stderr)\n if rc._spawned_and_waited and not call_args[\"return_cmd\"]:\n return str(rc)\n else:\n return rc\n\n\ndef compile_args(a, kwargs, sep, prefix):\n \"\"\"takes args and kwargs, as they were passed into the command instance\n being executed with __call__, and compose them into a flat list that\n will eventually be fed into exec. example:\n\n with this call:\n\n sh.ls(\"-l\", \"/tmp\", color=\"never\")\n\n this function receives\n\n args = ['-l', '/tmp']\n kwargs = {'color': 'never'}\n\n and produces\n\n ['-l', '/tmp', '--color=geneticnever']\n\n \"\"\"\n processed_args = []\n\n # aggregate positional args\n for arg in a:\n if isinstance(arg, (list, tuple)):\n if isinstance(arg, GlobResults) and not arg:\n arg = [arg.path]\n\n for sub_arg in arg:\n processed_args.append(sub_arg)\n elif isinstance(arg, dict):\n processed_args += _aggregate_keywords(arg, sep, prefix, raw=True)\n\n # see https://github.com/amoffat/sh/issues/522\n elif arg is None or arg is False:\n pass\n else:\n processed_args.append(str(arg))\n\n # aggregate the keyword arguments\n processed_args += _aggregate_keywords(kwargs, sep, prefix)\n\n return processed_args\n\n\ndef _aggregate_keywords(keywords, sep, prefix, raw=False):\n \"\"\"take our keyword arguments, and a separator, and compose the list of\n flat long (and short) arguments. example\n\n {'color': 'never', 't': True, 'something': True} with sep '='\n\n becomes\n\n ['--color=never', '-t', '--something']\n\n the `raw` argument indicates whether or not we should leave the argument\n name alone, or whether we should replace \"_\" with \"-\". if we pass in a\n dictionary, like this:\n\n sh.command({\"some_option\": 12})\n\n then `raw` gets set to True, because we want to leave the key as-is, to\n produce:\n\n ['--some_option=12']\n\n but if we just use a command's kwargs, `raw` is False, which means this:\n\n sh.command(some_option=12)\n\n becomes:\n\n ['--some-option=12']\n\n essentially, using kwargs is a convenience, but it lacks the ability to\n put a '-' in the name, so we do the replacement of '_' to '-' for you.\n but when you really don't want that to happen, you should use a\n dictionary instead with the exact names you want\n \"\"\"\n\n processed = []\n\n for k, maybe_list_of_v in keywords.items():\n # turn our value(s) into a list of values so that we can process them\n # all individually under the same key\n list_of_v = [maybe_list_of_v]\n if isinstance(maybe_list_of_v, (list, tuple)):\n list_of_v = maybe_list_of_v\n\n for v in list_of_v:\n # we're passing a short arg as a kwarg, example:\n # cut(d=\"\\t\")\n if len(k) == 1:\n if v is not False:\n processed.append(\"-\" + k)\n if v is not True:\n processed.append(str(v))\n\n # we're doing a long arg\n else:\n if not raw:\n k = k.replace(\"_\", \"-\")\n\n # if it's true, it has no value, just pass the name\n if v is True:\n processed.append(prefix + k)\n # if it's false, skip passing it\n elif v is False:\n pass\n\n # we may need to break the argument up into multiple arguments\n elif sep is None or sep == \" \":\n processed.append(prefix + k)\n processed.append(str(v))\n # otherwise just join it together into a single argument\n else:\n arg = f\"{prefix}{k}{sep}{v}\"\n processed.append(arg)\n\n return processed\n\n\ndef _start_daemon_thread(fn, name, exc_queue, *a):\n def wrap(*rgs, **kwargs):\n try:\n fn(*rgs, **kwargs)\n except Exception as e:\n exc_queue.put(e)\n raise\n\n thread = threading.Thread(target=wrap, name=name, args=a)\n thread.daemon = True\n thread.start()\n return thread\n\n\ndef setwinsize(fd, rows_cols):\n \"\"\"set the terminal size of a tty file descriptor. borrowed logic\n from pexpect.py\"\"\"\n rows, cols = rows_cols\n winsize = getattr(termios, \"TIOCSWINSZ\", -2146929561)\n\n s = struct.pack(\"HHHH\", rows, cols, 0, 0)\n fcntl.ioctl(fd, winsize, s)\n\n\ndef construct_streamreader_callback(process, handler):\n \"\"\"here we're constructing a closure for our streamreader callback. this\n is used in the case that we pass a callback into _out or _err, meaning we\n want to our callback to handle each bit of output\n\n we construct the closure based on how many arguments it takes. the reason\n for this is to make it as easy as possible for people to use, without\n limiting them. a new user will assume the callback takes 1 argument (the\n data). as they get more advanced, they may want to terminate the process,\n or pass some stdin back, and will realize that they can pass a callback of\n more args\"\"\"\n\n # implied arg refers to the \"self\" that methods will pass in. we need to\n # account for this implied arg when figuring out what function the user\n # passed in based on number of args\n implied_arg = 0\n\n partial_args = 0\n handler_to_inspect = handler\n\n if isinstance(handler, partial):\n partial_args = len(handler.args)\n handler_to_inspect = handler.func\n\n if inspect.ismethod(handler_to_inspect):\n implied_arg = 1\n num_args = get_num_args(handler_to_inspect)\n\n else:\n if inspect.isfunction(handler_to_inspect):\n num_args = get_num_args(handler_to_inspect)\n\n # is an object instance with __call__ method\n else:\n implied_arg = 1\n num_args = get_num_args(handler_to_inspect.__call__)\n\n net_args = num_args - implied_arg - partial_args\n\n handler_args = ()\n\n # just the chunk\n if net_args == 1:\n handler_args = ()\n\n # chunk, stdin\n if net_args == 2:\n handler_args = (process.stdin,)\n\n # chunk, stdin, process\n elif net_args == 3:\n # notice we're only storing a weakref, to prevent cyclic references\n # (where the process holds a streamreader, and a streamreader holds a\n # handler-closure with a reference to the process\n handler_args = (process.stdin, weakref.ref(process))\n\n def fn(chunk):\n # this is pretty ugly, but we're evaluating the process at call-time,\n # because it's a weakref\n a = handler_args\n if len(a) == 2:\n a = (handler_args[0], handler_args[1]())\n return handler(chunk, *a)\n\n return fn\n\n\ndef get_exc_exit_code_would_raise(exit_code, ok_codes, sigpipe_ok):\n exc = None\n success = exit_code in ok_codes\n signals_that_should_throw_exception = [\n sig for sig in SIGNALS_THAT_SHOULD_THROW_EXCEPTION if -sig not in ok_codes\n ]\n bad_sig = -exit_code in signals_that_should_throw_exception\n\n # if this is a piped command, SIGPIPE must be ignored by us and not raise an\n # exception, since it's perfectly normal for the consumer of a process's\n # pipe to terminate early\n if sigpipe_ok and -exit_code == signal.SIGPIPE:\n bad_sig = False\n success = True\n\n if not success or bad_sig:\n exc = get_rc_exc(exit_code)\n return exc\n\n\ndef handle_process_exit_code(exit_code):\n \"\"\"this should only ever be called once for each child process\"\"\"\n # if we exited from a signal, let our exit code reflect that\n if os.WIFSIGNALED(exit_code):\n exit_code = -os.WTERMSIG(exit_code)\n # otherwise just give us a normal exit code\n elif os.WIFEXITED(exit_code):\n exit_code = os.WEXITSTATUS(exit_code)\n else:\n raise RuntimeError(\"Unknown child exit status!\")\n\n return exit_code\n\n\ndef no_interrupt(syscall, *args, **kwargs):\n \"\"\"a helper for making system calls immune to EINTR\"\"\"\n ret = None\n\n while True:\n try:\n ret = syscall(*args, **kwargs)\n except OSError as e:\n if e.errno == errno.EINTR:\n continue\n else:\n raise\n else:\n break\n\n return ret\n\n\nclass OProc:\n \"\"\"this class is instantiated by RunningCommand for a command to be exec'd.\n it handles all the nasty business involved with correctly setting up the\n input/output to the child process. it gets its name for subprocess.Popen\n (process open) but we're calling ours OProc (open process)\"\"\"\n\n _default_window_size = (24, 80)\n\n # used in redirecting\n STDOUT = -1\n STDERR = -2\n\n def __init__(\n self,\n command,\n parent_log,\n cmd,\n stdin,\n stdout,\n stderr,\n call_args,\n pipe,\n process_assign_lock,\n ):\n \"\"\"\n cmd is the full list of arguments that will be exec'd. it includes the program\n name and all its arguments.\n\n stdin, stdout, stderr are what the child will use for standard input/output/err.\n\n call_args is a mapping of all the special keyword arguments to apply to the\n child process.\n \"\"\"\n self.command = command\n self.call_args = call_args\n\n # convenience\n ca = self.call_args\n\n if ca[\"uid\"] is not None:\n if os.getuid() != 0:\n raise RuntimeError(\"UID setting requires root privileges\")\n\n target_uid = ca[\"uid\"]\n\n pwrec = pwd.getpwuid(ca[\"uid\"])\n target_gid = pwrec.pw_gid\n else:\n target_uid, target_gid = None, None\n\n # I had issues with getting 'Input/Output error reading stdin' from dd,\n # until I set _tty_out=False\n if ca[\"piped\"]:\n ca[\"tty_out\"] = False\n\n self._stdin_process = None\n\n # if the objects that we are passing to the OProc happen to be a\n # file-like object that is a tty, for example `sys.stdin`, then, later\n # on in this constructor, we're going to skip out on setting up pipes\n # and pseudoterminals for those endpoints\n stdin_is_fd_based = ob_is_fd_based(stdin)\n stdout_is_fd_based = ob_is_fd_based(stdout)\n stderr_is_fd_based = ob_is_fd_based(stderr)\n\n if isinstance(ca[\"tee\"], (str, bool, int)) or ca[\"tee\"] is None:\n tee = {ca[\"tee\"]}\n else:\n tee = set(ca[\"tee\"])\n tee_out = TEE_STDOUT.intersection(tee)\n tee_err = TEE_STDERR.intersection(tee)\n\n single_tty = ca[\"tty_in\"] and ca[\"tty_out\"] and ca[\"unify_ttys\"]\n\n # this logic is a little convoluted, but basically this top-level\n # if/else is for consolidating input and output TTYs into a single\n # TTY. this is the only way some secure programs like ssh will\n # output correctly (is if stdout and stdin are both the same TTY)\n if single_tty:\n # master_fd, slave_fd = pty.openpty()\n #\n # Anything that is written on the master end is provided to the process on\n # the slave end as though it was\n # input typed on a terminal. -\"man 7 pty\"\n #\n # later, in the child process, we're going to do this, so keep it in mind:\n #\n # os.dup2(self._stdin_child_fd, 0)\n # os.dup2(self._stdout_child_fd, 1)\n # os.dup2(self._stderr_child_fd, 2)\n self._stdin_parent_fd, self._stdin_child_fd = pty.openpty()\n\n # this makes our parent fds behave like a terminal. it says that the very\n # same fd that we \"type\" to (for stdin) is the same one that we see output\n # printed to (for stdout)\n self._stdout_parent_fd = os.dup(self._stdin_parent_fd)\n\n # this line is what makes stdout and stdin attached to the same pty. in\n # other words the process will write to the same underlying fd as stdout\n # as it uses to read from for stdin. this makes programs like ssh happy\n self._stdout_child_fd = os.dup(self._stdin_child_fd)\n\n self._stderr_parent_fd = os.dup(self._stdin_parent_fd)\n self._stderr_child_fd = os.dup(self._stdin_child_fd)\n\n # do not consolidate stdin and stdout. this is the most common use-\n # case\n else:\n # this check here is because we may be doing piping and so our stdin\n # might be an instance of OProc\n if isinstance(stdin, OProc) and stdin.call_args[\"piped\"]:\n self._stdin_child_fd = stdin._pipe_fd\n self._stdin_parent_fd = None\n self._stdin_process = stdin\n\n elif stdin_is_fd_based:\n self._stdin_child_fd = os.dup(get_fileno(stdin))\n self._stdin_parent_fd = None\n\n elif ca[\"tty_in\"]:\n self._stdin_parent_fd, self._stdin_child_fd = pty.openpty()\n\n # tty_in=False is the default\n else:\n self._stdin_child_fd, self._stdin_parent_fd = os.pipe()\n\n if stdout_is_fd_based and not tee_out:\n self._stdout_child_fd = os.dup(get_fileno(stdout))\n self._stdout_parent_fd = None\n\n # tty_out=True is the default\n elif ca[\"tty_out\"]:\n self._stdout_parent_fd, self._stdout_child_fd = pty.openpty()\n\n else:\n self._stdout_parent_fd, self._stdout_child_fd = os.pipe()\n\n # unless STDERR is going to STDOUT, it ALWAYS needs to be a pipe,\n # and never a PTY. the reason for this is not totally clear to me,\n # but it has to do with the fact that if STDERR isn't set as the\n # CTTY (because STDOUT is), the STDERR buffer won't always flush\n # by the time the process exits, and the data will be lost.\n # i've only seen this on OSX.\n if stderr is OProc.STDOUT:\n # if stderr is going to stdout, but stdout is a tty or a pipe,\n # we should not specify a read_fd, because stdout is os.dup'ed\n # directly to the stdout fd (no pipe), and so stderr won't have\n # a slave end of a pipe either to dup\n if stdout_is_fd_based and not tee_out:\n self._stderr_parent_fd = None\n else:\n self._stderr_parent_fd = os.dup(self._stdout_parent_fd)\n self._stderr_child_fd = os.dup(self._stdout_child_fd)\n\n elif stderr_is_fd_based and not tee_err:\n self._stderr_child_fd = os.dup(get_fileno(stderr))\n self._stderr_parent_fd = None\n\n else:\n self._stderr_parent_fd, self._stderr_child_fd = os.pipe()\n\n piped = ca[\"piped\"]\n self._pipe_fd = None\n if piped:\n fd_to_use = self._stdout_parent_fd\n if piped == \"err\":\n fd_to_use = self._stderr_parent_fd\n self._pipe_fd = os.dup(fd_to_use)\n\n new_session = ca[\"new_session\"]\n new_group = ca[\"new_group\"]\n needs_ctty = ca[\"tty_in\"]\n\n # if we need a controlling terminal, we have to be in a new session where we\n # are the session leader, otherwise we would need to take over the existing\n # process session, and we can't do that(?)\n if needs_ctty:\n new_session = True\n\n self.ctty = None\n if needs_ctty:\n self.ctty = os.ttyname(self._stdin_child_fd)\n\n gc_enabled = gc.isenabled()\n if gc_enabled:\n gc.disable()\n\n # for synchronizing\n session_pipe_read, session_pipe_write = os.pipe()\n exc_pipe_read, exc_pipe_write = os.pipe()\n\n # this pipe is for synchronizing with the child that the parent has\n # closed its in/out/err fds. this is a bug on OSX (but not linux),\n # where we can lose output sometimes, due to a race, if we do\n # os.close(self._stdout_child_fd) in the parent after the child starts\n # writing.\n if IS_MACOS:\n close_pipe_read, close_pipe_write = os.pipe()\n else:\n close_pipe_read, close_pipe_write = None, None\n\n # session id, group id, process id\n self.sid = None\n self.pgid = None\n self.pid = os.fork()\n\n # child\n if self.pid == 0: # pragma: no cover\n if IS_MACOS:\n os.read(close_pipe_read, 1)\n os.close(close_pipe_read)\n os.close(close_pipe_write)\n\n # this is critical\n # our exc_pipe_write must have CLOEXEC enabled. the reason for this is\n # tricky: if our child (the block we're in now), has an exception, we need\n # to be able to write to exc_pipe_write, so that when the parent does\n # os.read(exc_pipe_read), it gets our traceback. however,\n # os.read(exc_pipe_read) in the parent blocks, so if our child *doesn't*\n # have an exception, and doesn't close the writing end, it hangs forever.\n # not good! but obviously the child can't close the writing end until it\n # knows it's not going to have an exception, which is impossible to know\n # because but what if os.execv has an exception? so the answer is CLOEXEC,\n # so that the writing end of the pipe gets closed upon successful exec,\n # and the parent reading the read end won't block (close breaks the block).\n flags = fcntl.fcntl(exc_pipe_write, fcntl.F_GETFD)\n flags |= fcntl.FD_CLOEXEC\n fcntl.fcntl(exc_pipe_write, fcntl.F_SETFD, flags)\n\n try:\n # ignoring SIGHUP lets us persist even after the controlling terminal\n # is closed\n if ca[\"bg\"] is True:\n signal.signal(signal.SIGHUP, signal.SIG_IGN)\n\n # python ignores SIGPIPE by default. we must make sure to put\n # this behavior back to the default for spawned processes,\n # otherwise SIGPIPE won't kill piped processes, which is what we\n # need, so that we can check the error code of the killed\n # process to see that SIGPIPE killed it\n signal.signal(signal.SIGPIPE, signal.SIG_DFL)\n\n # put our forked process in a new session? this will relinquish\n # any control of our inherited CTTY and also make our parent\n # process init\n if new_session:\n os.setsid()\n elif new_group:\n os.setpgid(0, 0)\n\n sid = os.getsid(0)\n pgid = os.getpgid(0)\n payload = (f\"{sid},{pgid}\").encode(DEFAULT_ENCODING)\n os.write(session_pipe_write, payload)\n\n if ca[\"tty_out\"] and not stdout_is_fd_based and not single_tty:\n # set raw mode, so there isn't any weird translation of\n # newlines to \\r\\n and other oddities. we're not outputting\n # to a terminal anyways\n #\n # we HAVE to do this here, and not in the parent process,\n # because we have to guarantee that this is set before the\n # child process is run, and we can't do it twice.\n tty.setraw(self._stdout_child_fd)\n\n # if the parent-side fd for stdin exists, close it. the case\n # where it may not exist is if we're using piping\n if self._stdin_parent_fd:\n os.close(self._stdin_parent_fd)\n\n if self._stdout_parent_fd:\n os.close(self._stdout_parent_fd)\n\n if self._stderr_parent_fd:\n os.close(self._stderr_parent_fd)\n\n os.close(session_pipe_read)\n os.close(exc_pipe_read)\n\n cwd = ca[\"cwd\"]\n if cwd:\n os.chdir(cwd)\n\n os.dup2(self._stdin_child_fd, 0)\n os.dup2(self._stdout_child_fd, 1)\n os.dup2(self._stderr_child_fd, 2)\n\n # set our controlling terminal, but only if we're using a tty\n # for stdin. it doesn't make sense to have a ctty otherwise\n if needs_ctty:\n tmp_fd = os.open(os.ttyname(0), os.O_RDWR)\n os.close(tmp_fd)\n\n if ca[\"tty_out\"] and not stdout_is_fd_based:\n setwinsize(1, ca[\"tty_size\"])\n\n if ca[\"uid\"] is not None:\n os.setgid(target_gid)\n os.setuid(target_uid)\n\n preexec_fn = ca[\"preexec_fn\"]\n if callable(preexec_fn):\n preexec_fn()\n\n close_fds = ca[\"close_fds\"]\n if ca[\"pass_fds\"]:\n close_fds = True\n\n if close_fds:\n pass_fds = {0, 1, 2, exc_pipe_write}\n pass_fds.update(ca[\"pass_fds\"])\n\n # don't inherit file descriptors\n try:\n inherited_fds = os.listdir(\"/dev/fd\")\n except OSError:\n # Some systems don't have /dev/fd. Raises OSError in\n # Python2, FileNotFoundError on Python3. The latter doesn't\n # exist on Python2, but inherits from IOError, which does.\n inherited_fds = os.listdir(\"/proc/self/fd\")\n inherited_fds = {int(fd) for fd in inherited_fds} - pass_fds\n for fd in inherited_fds:\n try:\n os.close(fd)\n except OSError:\n pass\n\n # python=3.6, locale=c will fail test_unicode_arg if we don't\n # explicitly encode to bytes via our desired encoding. this does\n # not seem to be the case in other python versions, even if locale=c\n bytes_cmd = [c.encode(ca[\"encoding\"]) for c in cmd]\n\n # actually execute the process\n if ca[\"env\"] is None:\n os.execv(bytes_cmd[0], bytes_cmd)\n else:\n os.execve(bytes_cmd[0], bytes_cmd, ca[\"env\"])\n\n # we must ensure that we carefully exit the child process on\n # exception, otherwise the parent process code will be executed\n # twice on exception https://github.com/amoffat/sh/issues/202\n #\n # if your parent process experiences an exit code 255, it is most\n # likely that an exception occurred between the fork of the child\n # and the exec. this should be reported.\n except Exception: # noqa: E722\n # some helpful debugging\n tb = traceback.format_exc().encode(\"utf8\", \"ignore\")\n\n try:\n os.write(exc_pipe_write, tb)\n\n except Exception as e:\n # dump to stderr if we cannot save it to exc_pipe_write\n sys.stderr.write(f\"\\nFATAL SH ERROR: {e}\\n\")\n\n finally:\n os._exit(255)\n\n # parent\n else:\n if gc_enabled:\n gc.enable()\n\n os.close(self._stdin_child_fd)\n os.close(self._stdout_child_fd)\n os.close(self._stderr_child_fd)\n\n # tell our child process that we've closed our write_fds, so it is\n # ok to proceed towards exec. see the comment where this pipe is\n # opened, for why this is necessary\n if IS_MACOS:\n os.close(close_pipe_read)\n os.write(close_pipe_write, str(1).encode(DEFAULT_ENCODING))\n os.close(close_pipe_write)\n\n os.close(exc_pipe_write)\n fork_exc = os.read(exc_pipe_read, 1024**2)\n os.close(exc_pipe_read)\n if fork_exc:\n fork_exc = fork_exc.decode(DEFAULT_ENCODING)\n raise ForkException(fork_exc)\n\n os.close(session_pipe_write)\n sid, pgid = (\n os.read(session_pipe_read, 1024).decode(DEFAULT_ENCODING).split(\",\")\n )\n os.close(session_pipe_read)\n self.sid = int(sid)\n self.pgid = int(pgid)\n\n # used to determine what exception to raise. if our process was\n # killed via a timeout counter, we'll raise something different than\n # a SIGKILL exception\n self.timed_out = False\n\n self.started = time.time()\n self.cmd = cmd\n\n # exit code should only be manipulated from within self._wait_lock\n # to prevent race conditions\n self.exit_code = None\n\n self.stdin = stdin\n\n # this accounts for when _out is a callable that is passed stdin. in that\n # case, if stdin is unspecified, we must set it to a queue, so callbacks can\n # put things on it\n if callable(ca[\"out\"]) and self.stdin is None:\n self.stdin = Queue()\n\n # _pipe_queue is used internally to hand off stdout from one process\n # to another. by default, all stdout from a process gets dumped\n # into this pipe queue, to be consumed in real time (hence the\n # thread-safe Queue), or at a potentially later time\n self._pipe_queue = Queue()\n\n # this is used to prevent a race condition when we're waiting for\n # a process to end, and the OProc's internal threads are also checking\n # for the processes's end\n self._wait_lock = threading.Lock()\n\n # these are for aggregating the stdout and stderr. we use a deque\n # because we don't want to overflow\n self._stdout = deque(maxlen=ca[\"internal_bufsize\"])\n self._stderr = deque(maxlen=ca[\"internal_bufsize\"])\n\n if ca[\"tty_in\"] and not stdin_is_fd_based:\n setwinsize(self._stdin_parent_fd, ca[\"tty_size\"])\n\n self.log = parent_log.get_child(\"process\", repr(self))\n\n self.log.debug(\"started process\")\n\n # disable echoing, but only if it's a tty that we created ourselves\n if ca[\"tty_in\"] and not stdin_is_fd_based:\n attr = termios.tcgetattr(self._stdin_parent_fd)\n attr[3] &= ~termios.ECHO\n termios.tcsetattr(self._stdin_parent_fd, termios.TCSANOW, attr)\n\n # this represents the connection from a Queue object (or whatever\n # we're using to feed STDIN) to the process's STDIN fd\n self._stdin_stream = None\n if self._stdin_parent_fd:\n log = self.log.get_child(\"streamwriter\", \"stdin\")\n self._stdin_stream = StreamWriter(\n log,\n self._stdin_parent_fd,\n self.stdin,\n ca[\"in_bufsize\"],\n ca[\"encoding\"],\n ca[\"tty_in\"],\n )\n\n stdout_pipe = None\n if pipe is OProc.STDOUT and not ca[\"no_pipe\"]:\n stdout_pipe = self._pipe_queue\n\n # this represents the connection from a process's STDOUT fd to\n # wherever it has to go, sometimes a pipe Queue (that we will use\n # to pipe data to other processes), and also an internal deque\n # that we use to aggregate all the output\n save_stdout = not ca[\"no_out\"] and (tee_out or stdout is None)\n\n pipe_out = ca[\"piped\"] in (\"out\", True)\n pipe_err = ca[\"piped\"] in (\"err\",)\n\n # if we're piping directly into another process's file descriptor, we\n # bypass reading from the stdout stream altogether, because we've\n # already hooked up this processes's stdout fd to the other\n # processes's stdin fd\n self._stdout_stream = None\n if not pipe_out and self._stdout_parent_fd:\n if callable(stdout):\n stdout = construct_streamreader_callback(self, stdout)\n self._stdout_stream = StreamReader(\n self.log.get_child(\"streamreader\", \"stdout\"),\n self._stdout_parent_fd,\n stdout,\n self._stdout,\n ca[\"out_bufsize\"],\n ca[\"encoding\"],\n ca[\"decode_errors\"],\n stdout_pipe,\n save_data=save_stdout,\n )\n\n elif self._stdout_parent_fd:\n os.close(self._stdout_parent_fd)\n\n # if stderr is going to one place (because it's grouped with stdout,\n # or we're dealing with a single tty), then we don't actually need a\n # stream reader for stderr, because we've already set one up for\n # stdout above\n self._stderr_stream = None\n if (\n stderr is not OProc.STDOUT\n and not single_tty\n and not pipe_err\n and self._stderr_parent_fd\n ):\n stderr_pipe = None\n if pipe is OProc.STDERR and not ca[\"no_pipe\"]:\n stderr_pipe = self._pipe_queue\n\n save_stderr = not ca[\"no_err\"] and (tee_err or stderr is None)\n\n if callable(stderr):\n stderr = construct_streamreader_callback(self, stderr)\n\n self._stderr_stream = StreamReader(\n Logger(\"streamreader\"),\n self._stderr_parent_fd,\n stderr,\n self._stderr,\n ca[\"err_bufsize\"],\n ca[\"encoding\"],\n ca[\"decode_errors\"],\n stderr_pipe,\n save_data=save_stderr,\n )\n\n elif self._stderr_parent_fd:\n os.close(self._stderr_parent_fd)\n\n def timeout_fn():\n self.timed_out = True\n try:\n self.signal(ca[\"timeout_signal\"])\n except ProcessLookupError:\n # Edge case: the process probably exited\n pass\n\n self._timeout_event = None\n self._timeout_timer = None\n if ca[\"timeout\"]:\n self._timeout_event = threading.Event()\n self._timeout_timer = threading.Timer(\n ca[\"timeout\"], self._timeout_event.set\n )\n self._timeout_timer.start()\n\n # this is for cases where we know that the RunningCommand that was\n # launched was not .wait()ed on to complete. in those unique cases,\n # we allow the thread that processes output to report exceptions in\n # that thread. it's important that we only allow reporting of the\n # exception, and nothing else (like the additional stuff that\n # RunningCommand.wait() does), because we want the exception to be\n # re-raised in the future, if we DO call .wait()\n handle_exit_code = None\n if (\n not self.command._spawned_and_waited\n and ca[\"bg_exc\"]\n # we don't want background exceptions if we're doing async stuff,\n # because we want those to bubble up.\n and not ca[\"async\"]\n ):\n\n def fn(exit_code):\n with process_assign_lock:\n return self.command.handle_command_exit_code(exit_code)\n\n handle_exit_code = fn\n\n self._quit_threads = threading.Event()\n\n thread_name = f\"background thread for pid {self.pid}\"\n self._bg_thread_exc_queue = Queue(1)\n self._background_thread = _start_daemon_thread(\n background_thread,\n thread_name,\n self._bg_thread_exc_queue,\n timeout_fn,\n self._timeout_event,\n handle_exit_code,\n self.is_alive,\n self._quit_threads,\n )\n\n # start the main io threads. stdin thread is not needed if we are\n # connecting from another process's stdout pipe\n self._input_thread = None\n self._input_thread_exc_queue = Queue(1)\n if self._stdin_stream:\n close_before_term = not needs_ctty\n thread_name = f\"STDIN thread for pid {self.pid}\"\n self._input_thread = _start_daemon_thread(\n input_thread,\n thread_name,\n self._input_thread_exc_queue,\n self.log,\n self._stdin_stream,\n self.is_alive,\n self._quit_threads,\n close_before_term,\n )\n\n # this event is for cases where the subprocess that we launch\n # launches its OWN subprocess and os.dup's the stdout/stderr fds to that\n # new subprocess. in that case, stdout and stderr will never EOF,\n # so our output_thread will never finish and will hang. this event\n # prevents that hanging\n self._stop_output_event = threading.Event()\n\n # we need to set up a callback to fire when our `output_thread` is about\n # to exit. this callback will set an asyncio Event, so that coroutiens can\n # be notified that our output is finished.\n # if the `sh` command was launched from within a thread (so we're not in\n # the main thread), then we won't have an event loop.\n try:\n loop = asyncio.get_running_loop()\n except RuntimeError:\n\n def output_complete():\n pass\n\n else:\n\n def output_complete():\n loop.call_soon_threadsafe(self.command.aio_output_complete.set)\n\n self._output_thread_exc_queue = Queue(1)\n thread_name = f\"STDOUT/ERR thread for pid {self.pid}\"\n self._output_thread = _start_daemon_thread(\n output_thread,\n thread_name,\n self._output_thread_exc_queue,\n self.log,\n self._stdout_stream,\n self._stderr_stream,\n self._timeout_event,\n self.is_alive,\n self._quit_threads,\n self._stop_output_event,\n output_complete,\n )\n\n def __repr__(self):\n return f\"\"\n\n def change_in_bufsize(self, buf):\n self._stdin_stream.stream_bufferer.change_buffering(buf)\n\n def change_out_bufsize(self, buf):\n self._stdout_stream.stream_bufferer.change_buffering(buf)\n\n def change_err_bufsize(self, buf):\n self._stderr_stream.stream_bufferer.change_buffering(buf)\n\n @property\n def stdout(self):\n return \"\".encode(self.call_args[\"encoding\"]).join(self._stdout)\n\n @property\n def stderr(self):\n return \"\".encode(self.call_args[\"encoding\"]).join(self._stderr)\n\n def get_pgid(self):\n \"\"\"return the CURRENT group id of the process. this differs from\n self.pgid in that this reflects the current state of the process, where\n self.pgid is the group id at launch\"\"\"\n return os.getpgid(self.pid)\n\n def get_sid(self):\n \"\"\"return the CURRENT session id of the process. this differs from\n self.sid in that this reflects the current state of the process, where\n self.sid is the session id at launch\"\"\"\n return os.getsid(self.pid)\n\n def signal_group(self, sig):\n self.log.debug(\"sending signal %d to group\", sig)\n os.killpg(self.get_pgid(), sig)\n\n def signal(self, sig):\n self.log.debug(\"sending signal %d\", sig)\n os.kill(self.pid, sig)\n\n def kill_group(self):\n self.log.debug(\"killing group\")\n self.signal_group(signal.SIGKILL)\n\n def kill(self):\n self.log.debug(\"killing\")\n self.signal(signal.SIGKILL)\n\n def terminate(self):\n self.log.debug(\"terminating\")\n self.signal(signal.SIGTERM)\n\n def is_alive(self):\n \"\"\"polls if our child process has completed, without blocking. this\n method has side-effects, such as setting our exit_code, if we happen to\n see our child exit while this is running\"\"\"\n\n if self.exit_code is not None:\n return False, self.exit_code\n\n # what we're doing here essentially is making sure that the main thread\n # (or another thread), isn't calling .wait() on the process. because\n # .wait() calls os.waitpid(self.pid, 0), we can't do an os.waitpid\n # here...because if we did, and the process exited while in this\n # thread, the main thread's os.waitpid(self.pid, 0) would raise OSError\n # (because the process ended in another thread).\n #\n # so essentially what we're doing is, using this lock, checking if\n # we're calling .wait(), and if we are, let .wait() get the exit code\n # and handle the status, otherwise let us do it.\n #\n # Using a small timeout provides backpressure against code that spams\n # calls to .is_alive() which may block the main thread from acquiring\n # the lock otherwise.\n acquired = self._wait_lock.acquire(timeout=0.00001)\n if not acquired:\n if self.exit_code is not None:\n return False, self.exit_code\n return True, self.exit_code\n\n witnessed_end = False\n try:\n # WNOHANG is just that...we're calling waitpid without hanging...\n # essentially polling the process. the return result is (0, 0) if\n # there's no process status, so we check that pid == self.pid below\n # in order to determine how to proceed\n pid, exit_code = no_interrupt(os.waitpid, self.pid, os.WNOHANG)\n if pid == self.pid:\n self.exit_code = handle_process_exit_code(exit_code)\n witnessed_end = True\n\n return False, self.exit_code\n\n # no child process\n except OSError:\n return False, self.exit_code\n else:\n return True, self.exit_code\n finally:\n self._wait_lock.release()\n if witnessed_end:\n self._process_just_ended()\n\n def _process_just_ended(self):\n if self._timeout_timer:\n self._timeout_timer.cancel()\n\n done_callback = self.call_args[\"done\"]\n if done_callback:\n success = self.exit_code in self.call_args[\"ok_code\"]\n done_callback(success, self.exit_code)\n\n # this can only be closed at the end of the process, because it might be\n # the CTTY, and closing it prematurely will send a SIGHUP. we also\n # don't want to close it if there's a self._stdin_stream, because that\n # is in charge of closing it also\n if self._stdin_parent_fd and not self._stdin_stream:\n os.close(self._stdin_parent_fd)\n\n def wait(self):\n \"\"\"waits for the process to complete, handles the exit code\"\"\"\n\n self.log.debug(\"acquiring wait lock to wait for completion\")\n # using the lock in a with-context blocks, which is what we want if\n # we're running wait()\n with self._wait_lock:\n self.log.debug(\"got wait lock\")\n witnessed_end = False\n\n if self.exit_code is None:\n self.log.debug(\"exit code not set, waiting on pid\")\n pid, exit_code = no_interrupt(os.waitpid, self.pid, 0) # blocks\n self.exit_code = handle_process_exit_code(exit_code)\n witnessed_end = True\n\n else:\n self.log.debug(\n \"exit code already set (%d), no need to wait\", self.exit_code\n )\n self._process_exit_cleanup(witnessed_end=witnessed_end)\n return self.exit_code\n\n def _process_exit_cleanup(self, witnessed_end):\n self._quit_threads.set()\n\n # we may not have a thread for stdin, if the pipe has been connected\n # via _piped=\"direct\"\n if self._input_thread:\n self._input_thread.join()\n\n # wait, then signal to our output thread that the child process is\n # done, and we should have finished reading all the stdout/stderr\n # data that we can by now\n timer = threading.Timer(2.0, self._stop_output_event.set)\n timer.start()\n\n # wait for our stdout and stderr streamreaders to finish reading and\n # aggregating the process output\n self._output_thread.join()\n timer.cancel()\n\n self._background_thread.join()\n\n if witnessed_end:\n self._process_just_ended()\n\n\ndef input_thread(log, stdin, is_alive, quit_thread, close_before_term):\n \"\"\"this is run in a separate thread. it writes into our process's\n stdin (a streamwriter) and waits the process to end AND everything that\n can be written to be written\"\"\"\n\n closed = False\n alive = True\n poller = Poller()\n poller.register_write(stdin)\n\n while poller and alive:\n changed = poller.poll(1)\n for fd, events in changed:\n if events & (POLLER_EVENT_WRITE | POLLER_EVENT_HUP):\n log.debug(\"%r ready for more input\", stdin)\n done = stdin.write()\n\n if done:\n poller.unregister(stdin)\n if close_before_term:\n stdin.close()\n closed = True\n\n alive, _ = is_alive()\n\n while alive:\n quit_thread.wait(1)\n alive, _ = is_alive()\n\n if not closed:\n stdin.close()\n\n\ndef event_wait(ev, timeout=None):\n triggered = ev.wait(timeout)\n return triggered\n\n\ndef background_thread(\n timeout_fn, timeout_event, handle_exit_code, is_alive, quit_thread\n):\n \"\"\"handles the timeout logic\"\"\"\n\n # if there's a timeout event, loop\n if timeout_event:\n while not quit_thread.is_set():\n timed_out = event_wait(timeout_event, 0.1)\n if timed_out:\n timeout_fn()\n break\n\n # handle_exit_code will be a function ONLY if our command was NOT waited on\n # as part of its spawning. in other words, it's probably a background\n # command\n #\n # this reports the exit code exception in our thread. it's purely for the\n # user's awareness, and cannot be caught or used in any way, so it's ok to\n # suppress this during the tests\n if handle_exit_code and not RUNNING_TESTS: # pragma: no cover\n alive = True\n exit_code = None\n while alive:\n quit_thread.wait(1)\n alive, exit_code = is_alive()\n\n handle_exit_code(exit_code)\n\n\ndef output_thread(\n log,\n stdout,\n stderr,\n timeout_event,\n is_alive,\n quit_thread,\n stop_output_event,\n output_complete,\n):\n \"\"\"this function is run in a separate thread. it reads from the\n process's stdout stream (a streamreader), and waits for it to claim that\n its done\"\"\"\n\n poller = Poller()\n if stdout is not None:\n poller.register_read(stdout)\n if stderr is not None:\n poller.register_read(stderr)\n\n # this is our poll loop for polling stdout or stderr that is ready to\n # be read and processed. if one of those streamreaders indicate that it\n # is done altogether being read from, we remove it from our list of\n # things to poll. when no more things are left to poll, we leave this\n # loop and clean up\n while poller:\n changed = no_interrupt(poller.poll, 0.1)\n for f, events in changed:\n if events & (POLLER_EVENT_READ | POLLER_EVENT_HUP):\n log.debug(\"%r ready to be read from\", f)\n done = f.read()\n if done:\n poller.unregister(f)\n elif events & POLLER_EVENT_ERROR:\n # for some reason, we have to just ignore streams that have had an\n # error. i'm not exactly sure why, but don't remove this until we\n # figure that out, and create a test for it\n pass\n\n if timeout_event and timeout_event.is_set():\n break\n\n if stop_output_event.is_set():\n break\n\n # we need to wait until the process is guaranteed dead before closing our\n # outputs, otherwise SIGPIPE\n alive, _ = is_alive()\n while alive:\n quit_thread.wait(1)\n alive, _ = is_alive()\n\n if stdout:\n stdout.close()\n\n if stderr:\n stderr.close()\n\n output_complete()\n\n\nclass DoneReadingForever(Exception):\n pass\n\n\nclass NotYetReadyToRead(Exception):\n pass\n\n\ndef determine_how_to_read_input(input_obj):\n \"\"\"given some kind of input object, return a function that knows how to\n read chunks of that input object.\n\n each reader function should return a chunk and raise a DoneReadingForever\n exception, or return None, when there's no more data to read\n\n NOTE: the function returned does not need to care much about the requested\n buffering type (eg, unbuffered vs newline-buffered). the StreamBufferer\n will take care of that. these functions just need to return a\n reasonably-sized chunk of data.\"\"\"\n\n if isinstance(input_obj, Queue):\n log_msg = \"queue\"\n get_chunk = get_queue_chunk_reader(input_obj)\n\n elif callable(input_obj):\n log_msg = \"callable\"\n get_chunk = get_callable_chunk_reader(input_obj)\n\n # also handles stringio\n elif hasattr(input_obj, \"read\"):\n log_msg = \"file descriptor\"\n get_chunk = get_file_chunk_reader(input_obj)\n\n elif isinstance(input_obj, str):\n log_msg = \"string\"\n get_chunk = get_iter_string_reader(input_obj)\n\n elif isinstance(input_obj, bytes):\n log_msg = \"bytes\"\n get_chunk = get_iter_string_reader(input_obj)\n\n elif isinstance(input_obj, GeneratorType):\n log_msg = \"generator\"\n get_chunk = get_iter_chunk_reader(iter(input_obj))\n\n elif input_obj is None:\n log_msg = \"None\"\n\n def raise_():\n raise DoneReadingForever\n\n get_chunk = raise_\n\n else:\n try:\n it = iter(input_obj)\n except TypeError:\n raise Exception(\"unknown input object\")\n else:\n log_msg = \"general iterable\"\n get_chunk = get_iter_chunk_reader(it)\n\n return get_chunk, log_msg\n\n\ndef get_queue_chunk_reader(stdin):\n def fn():\n try:\n chunk = stdin.get(True, 0.1)\n except Empty:\n raise NotYetReadyToRead\n if chunk is None:\n raise DoneReadingForever\n return chunk\n\n return fn\n\n\ndef get_callable_chunk_reader(stdin):\n def fn():\n try:\n data = stdin()\n except DoneReadingForever:\n raise\n\n if not data:\n raise DoneReadingForever\n\n return data\n\n return fn\n\n\ndef get_iter_string_reader(stdin):\n \"\"\"return an iterator that returns a chunk of a string every time it is\n called. notice that even though bufsize_type might be line buffered, we're\n not doing any line buffering here. that's because our StreamBufferer\n handles all buffering. we just need to return a reasonable-sized chunk.\"\"\"\n bufsize = 1024\n iter_str = (stdin[i : i + bufsize] for i in range(0, len(stdin), bufsize))\n return get_iter_chunk_reader(iter_str)\n\n\ndef get_iter_chunk_reader(stdin):\n def fn():\n try:\n chunk = stdin.__next__()\n return chunk\n except StopIteration:\n raise DoneReadingForever\n\n return fn\n\n\ndef get_file_chunk_reader(stdin):\n bufsize = 1024\n\n def fn():\n # python 3.* includes a fileno on stringios, but accessing it throws an\n # exception. that exception is how we'll know we can't do a poll on\n # stdin\n is_real_file = True\n try:\n stdin.fileno()\n except UnsupportedOperation:\n is_real_file = False\n\n # this poll is for files that may not yet be ready to read. we test\n # for fileno because StringIO/BytesIO cannot be used in a poll\n if is_real_file and hasattr(stdin, \"fileno\"):\n poller = Poller()\n poller.register_read(stdin)\n changed = poller.poll(0.1)\n ready = False\n for fd, events in changed:\n if events & (POLLER_EVENT_READ | POLLER_EVENT_HUP):\n ready = True\n if not ready:\n raise NotYetReadyToRead\n\n chunk = stdin.read(bufsize)\n if not chunk:\n raise DoneReadingForever\n else:\n return chunk\n\n return fn\n\n\ndef bufsize_type_to_bufsize(bf_type):\n \"\"\"for a given bufsize type, return the actual bufsize we will read.\n notice that although 1 means \"newline-buffered\", we're reading a chunk size\n of 1024. this is because we have to read something. we let a\n StreamBufferer instance handle splitting our chunk on newlines\"\"\"\n\n # newlines\n if bf_type == 1:\n bufsize = 1024\n # unbuffered\n elif bf_type == 0:\n bufsize = 1\n # or buffered by specific amount\n else:\n bufsize = bf_type\n\n return bufsize\n\n\nclass StreamWriter:\n \"\"\"StreamWriter reads from some input (the stdin param) and writes to a fd\n (the stream param). the stdin may be a Queue, a callable, something with\n the \"read\" method, a string, or an iterable\"\"\"\n\n def __init__(self, log, stream, stdin, bufsize_type, encoding, tty_in):\n self.stream = stream\n self.stdin = stdin\n\n self.log = log\n self.encoding = encoding\n self.tty_in = tty_in\n\n self.stream_bufferer = StreamBufferer(bufsize_type, self.encoding)\n self.get_chunk, log_msg = determine_how_to_read_input(stdin)\n self.log.debug(\"parsed stdin as a %s\", log_msg)\n\n def fileno(self):\n \"\"\"defining this allows us to do poll on an instance of this\n class\"\"\"\n return self.stream\n\n def write(self):\n \"\"\"attempt to get a chunk of data to write to our child process's\n stdin, then write it. the return value answers the questions \"are we\n done writing forever?\" \"\"\"\n\n # get_chunk may sometimes return bytes, and sometimes return strings\n # because of the nature of the different types of STDIN objects we\n # support\n try:\n chunk = self.get_chunk()\n if chunk is None:\n raise DoneReadingForever\n\n except DoneReadingForever:\n self.log.debug(\"done reading\")\n\n if self.tty_in:\n # EOF time\n try:\n char = termios.tcgetattr(self.stream)[6][termios.VEOF]\n except: # noqa: E722\n char = chr(4).encode()\n\n # normally, one EOF should be enough to signal to an program\n # that is read()ing, to return 0 and be on your way. however,\n # some programs are misbehaved, like python3.1 and python3.2.\n # they don't stop reading sometimes after read() returns 0.\n # this can be demonstrated with the following program:\n #\n # import sys\n # sys.stdout.write(sys.stdin.read())\n #\n # then type 'a' followed by ctrl-d 3 times. in python\n # 2.6,2.7,3.3,3.4,3.5,3.6, it only takes 2 ctrl-d to terminate.\n # however, in python 3.1 and 3.2, it takes all 3.\n #\n # so here we send an extra EOF along, just in case. i don't\n # believe it can hurt anything\n os.write(self.stream, char)\n os.write(self.stream, char)\n\n return True\n\n except NotYetReadyToRead:\n self.log.debug(\"received no data\")\n return False\n\n # if we're not bytes, make us bytes\n if not isinstance(chunk, bytes):\n chunk = chunk.encode(self.encoding)\n\n for proc_chunk in self.stream_bufferer.process(chunk):\n self.log.debug(\"got chunk size %d: %r\", len(proc_chunk), proc_chunk[:30])\n\n self.log.debug(\"writing chunk to process\")\n try:\n os.write(self.stream, proc_chunk)\n except OSError:\n self.log.debug(\"OSError writing stdin chunk\")\n return True\n\n def close(self):\n self.log.debug(\"closing, but flushing first\")\n chunk = self.stream_bufferer.flush()\n self.log.debug(\"got chunk size %d to flush: %r\", len(chunk), chunk[:30])\n try:\n if chunk:\n os.write(self.stream, chunk)\n\n except OSError:\n pass\n\n os.close(self.stream)\n\n\ndef determine_how_to_feed_output(handler, encoding, decode_errors):\n if callable(handler):\n process, finish = get_callback_chunk_consumer(handler, encoding, decode_errors)\n\n # in py3, this is used for bytes\n elif isinstance(handler, BytesIO):\n process, finish = get_cstringio_chunk_consumer(handler)\n\n # in py3, this is used for unicode\n elif isinstance(handler, StringIO):\n process, finish = get_stringio_chunk_consumer(handler, encoding, decode_errors)\n\n elif hasattr(handler, \"write\"):\n process, finish = get_file_chunk_consumer(handler, decode_errors)\n\n else:\n try:\n handler = int(handler)\n except (ValueError, TypeError):\n\n def process(chunk):\n return False # noqa: E731\n\n def finish():\n return None # noqa: E731\n\n else:\n process, finish = get_fd_chunk_consumer(handler, decode_errors)\n\n return process, finish\n\n\ndef get_fd_chunk_consumer(handler, decode_errors):\n handler = fdopen(handler, \"w\", closefd=False)\n return get_file_chunk_consumer(handler, decode_errors)\n\n\ndef get_file_chunk_consumer(handler, decode_errors):\n if getattr(handler, \"encoding\", None):\n\n def encode(chunk):\n return chunk.decode(handler.encoding, decode_errors) # noqa: E731\n\n else:\n\n def encode(chunk):\n return chunk # noqa: E731\n\n if hasattr(handler, \"flush\"):\n flush = handler.flush\n else:\n\n def flush():\n return None # noqa: E731\n\n def process(chunk):\n handler.write(encode(chunk))\n # we should flush on an fd. chunk is already the correctly-buffered\n # size, so we don't need the fd buffering as well\n flush()\n return False\n\n def finish():\n flush()\n\n return process, finish\n\n\ndef get_callback_chunk_consumer(handler, encoding, decode_errors):\n def process(chunk):\n # try to use the encoding first, if that doesn't work, send\n # the bytes, because it might be binary\n try:\n chunk = chunk.decode(encoding, decode_errors)\n except UnicodeDecodeError:\n pass\n return handler(chunk)\n\n def finish():\n pass\n\n return process, finish\n\n\ndef get_cstringio_chunk_consumer(handler):\n def process(chunk):\n handler.write(chunk)\n return False\n\n def finish():\n pass\n\n return process, finish\n\n\ndef get_stringio_chunk_consumer(handler, encoding, decode_errors):\n def process(chunk):\n handler.write(chunk.decode(encoding, decode_errors))\n return False\n\n def finish():\n pass\n\n return process, finish\n\n\nclass StreamReader:\n \"\"\"reads from some output (the stream) and sends what it just read to the\n handler.\"\"\"\n\n def __init__(\n self,\n log,\n stream,\n handler,\n buffer,\n bufsize_type,\n encoding,\n decode_errors,\n pipe_queue=None,\n save_data=True,\n ):\n self.stream = stream\n self.buffer = buffer\n self.save_data = save_data\n self.encoding = encoding\n self.decode_errors = decode_errors\n\n self.pipe_queue = None\n if pipe_queue:\n self.pipe_queue = weakref.ref(pipe_queue)\n\n self.log = log\n\n self.stream_bufferer = StreamBufferer(\n bufsize_type, self.encoding, self.decode_errors\n )\n self.bufsize = bufsize_type_to_bufsize(bufsize_type)\n\n self.process_chunk, self.finish_chunk_processor = determine_how_to_feed_output(\n handler, encoding, decode_errors\n )\n\n self.should_quit = False\n\n def fileno(self):\n \"\"\"defining this allows us to do poll on an instance of this\n class\"\"\"\n return self.stream\n\n def close(self):\n chunk = self.stream_bufferer.flush()\n self.log.debug(\"got chunk size %d to flush: %r\", len(chunk), chunk[:30])\n if chunk:\n self.write_chunk(chunk)\n\n self.finish_chunk_processor()\n\n if self.pipe_queue and self.save_data:\n self.pipe_queue().put(None)\n\n os.close(self.stream)\n\n def write_chunk(self, chunk):\n # in PY3, the chunk coming in will be bytes, so keep that in mind\n\n if not self.should_quit:\n self.should_quit = self.process_chunk(chunk)\n\n if self.save_data:\n self.buffer.append(chunk)\n\n if self.pipe_queue:\n self.log.debug(\"putting chunk onto pipe: %r\", chunk[:30])\n self.pipe_queue().put(chunk)\n\n def read(self):\n # if we're PY3, we're reading bytes, otherwise we're reading\n # str\n try:\n chunk = no_interrupt(os.read, self.stream, self.bufsize)\n except OSError as e:\n self.log.debug(\"got errno %d, done reading\", e.errno)\n return True\n if not chunk:\n self.log.debug(\"got no chunk, done reading\")\n return True\n\n self.log.debug(\"got chunk size %d: %r\", len(chunk), chunk[:30])\n for chunk in self.stream_bufferer.process(chunk):\n self.write_chunk(chunk)\n\n\nclass StreamBufferer:\n \"\"\"this is used for feeding in chunks of stdout/stderr, and breaking it up\n into chunks that will actually be put into the internal buffers. for\n example, if you have two processes, one being piped to the other, and you\n want that, first process to feed lines of data (instead of the chunks\n however they come in), OProc will use an instance of this class to chop up\n the data and feed it as lines to be sent down the pipe\"\"\"\n\n def __init__(self, buffer_type, encoding=DEFAULT_ENCODING, decode_errors=\"strict\"):\n # 0 for unbuffered, 1 for line, everything else for that amount\n self.type = buffer_type\n self.buffer = []\n self.n_buffer_count = 0\n self.encoding = encoding\n self.decode_errors = decode_errors\n\n # this is for if we change buffering types. if we change from line\n # buffered to unbuffered, its very possible that our self.buffer list\n # has data that was being saved up (while we searched for a newline).\n # we need to use that up, so we don't lose it\n self._use_up_buffer_first = False\n\n # the buffering lock is used because we might change the buffering\n # types from a different thread. for example, if we have a stdout\n # callback, we might use it to change the way stdin buffers. so we\n # lock\n self._buffering_lock = threading.RLock()\n self.log = Logger(\"stream_bufferer\")\n\n def change_buffering(self, new_type):\n # TODO, when we stop supporting 2.6, make this a with context\n self.log.debug(\"acquiring buffering lock for changing buffering\")\n self._buffering_lock.acquire()\n self.log.debug(\"got buffering lock for changing buffering\")\n try:\n if new_type == 0:\n self._use_up_buffer_first = True\n\n self.type = new_type\n finally:\n self._buffering_lock.release()\n self.log.debug(\"released buffering lock for changing buffering\")\n\n def process(self, chunk):\n # MAKE SURE THAT THE INPUT IS PY3 BYTES\n # THE OUTPUT IS ALWAYS PY3 BYTES\n\n # TODO, when we stop supporting 2.6, make this a with context\n self.log.debug(\n \"acquiring buffering lock to process chunk (buffering: %d)\", self.type\n )\n self._buffering_lock.acquire()\n self.log.debug(\"got buffering lock to process chunk (buffering: %d)\", self.type)\n try:\n # unbuffered\n if self.type == 0:\n if self._use_up_buffer_first:\n self._use_up_buffer_first = False\n to_write = self.buffer\n self.buffer = []\n to_write.append(chunk)\n return to_write\n\n return [chunk]\n\n # line buffered\n elif self.type == 1:\n total_to_write = []\n nl = \"\\n\".encode(self.encoding)\n while True:\n newline = chunk.find(nl)\n if newline == -1:\n break\n\n chunk_to_write = chunk[: newline + 1]\n if self.buffer:\n chunk_to_write = b\"\".join(self.buffer) + chunk_to_write\n\n self.buffer = []\n self.n_buffer_count = 0\n\n chunk = chunk[newline + 1 :]\n total_to_write.append(chunk_to_write)\n\n if chunk:\n self.buffer.append(chunk)\n self.n_buffer_count += len(chunk)\n return total_to_write\n\n # N size buffered\n else:\n total_to_write = []\n while True:\n overage = self.n_buffer_count + len(chunk) - self.type\n if overage >= 0:\n ret = \"\".encode(self.encoding).join(self.buffer) + chunk\n chunk_to_write = ret[: self.type]\n chunk = ret[self.type :]\n total_to_write.append(chunk_to_write)\n self.buffer = []\n self.n_buffer_count = 0\n else:\n self.buffer.append(chunk)\n self.n_buffer_count += len(chunk)\n break\n return total_to_write\n finally:\n self._buffering_lock.release()\n self.log.debug(\n \"released buffering lock for processing chunk (buffering: %d)\",\n self.type,\n )\n\n def flush(self):\n self.log.debug(\"acquiring buffering lock for flushing buffer\")\n self._buffering_lock.acquire()\n self.log.debug(\"got buffering lock for flushing buffer\")\n try:\n ret = \"\".encode(self.encoding).join(self.buffer)\n self.buffer = []\n return ret\n finally:\n self._buffering_lock.release()\n self.log.debug(\"released buffering lock for flushing buffer\")\n\n\ndef with_lock(lock):\n def wrapped(fn):\n fn = contextmanager(fn)\n\n @contextmanager\n def wrapped2(*args, **kwargs):\n with lock:\n with fn(*args, **kwargs):\n yield\n\n return wrapped2\n\n return wrapped\n\n\n@with_lock(PUSHD_LOCK)\ndef pushd(path):\n \"\"\"pushd changes the actual working directory for the duration of the\n context, unlike the _cwd arg this will work with other built-ins such as\n sh.glob correctly\"\"\"\n orig_path = os.getcwd()\n os.chdir(path)\n try:\n yield\n finally:\n os.chdir(orig_path)\n\n\n@contextmanager\ndef _args(**kwargs):\n \"\"\"allows us to temporarily override all the special keyword parameters in\n a with context\"\"\"\n\n kwargs_str = \",\".join([f\"{k}={v!r}\" for k, v in kwargs.items()])\n\n raise DeprecationWarning(\n f\"\"\"\n\nsh.args() has been deprecated because it was never thread safe. use the\nfollowing instead:\n\n sh2 = sh({kwargs_str})\n sh2.your_command()\n\nor\n\n sh2 = sh({kwargs_str})\n from sh2 import your_command\n your_command()\n\n\"\"\"\n )\n\n\nclass Environment(dict):\n \"\"\"this allows lookups to names that aren't found in the global scope to be\n searched for as a program name. for example, if \"ls\" isn't found in this\n module's scope, we consider it a system program and try to find it.\n\n we use a dict instead of just a regular object as the base class because the\n exec() statement used in the run_repl requires the \"globals\" argument to be a\n dictionary\"\"\"\n\n # this is a list of all of the names that the sh module exports that will\n # not resolve to functions. we don't want to accidentally shadow real\n # commands with functions/imports that we define in sh.py. for example,\n # \"import time\" may override the time system program\n allowlist = {\n \"Command\",\n \"RunningCommand\",\n \"CommandNotFound\",\n \"DEFAULT_ENCODING\",\n \"DoneReadingForever\",\n \"ErrorReturnCode\",\n \"NotYetReadyToRead\",\n \"SignalException\",\n \"ForkException\",\n \"TimeoutException\",\n \"StreamBufferer\",\n \"_aggregate_keywords\",\n \"__project_url__\",\n \"__version__\",\n \"__file__\",\n \"_args\",\n \"pushd\",\n \"glob\",\n \"contrib\",\n }\n\n def __init__(self, globs, baked_args=None):\n \"\"\"baked_args are defaults for the 'sh' execution context. for\n example:\n\n tmp = sh(_out=StringIO())\n\n 'out' would end up in here as an entry in the baked_args dict\"\"\"\n super(dict, self).__init__()\n self.globs = globs\n self.baked_args = baked_args or {}\n\n def __getitem__(self, k):\n if k == \"args\":\n # Let the deprecated '_args' context manager be imported as 'args'\n k = \"_args\"\n\n # if we're trying to import something real, see if it's in our global scope.\n # what defines \"real\" is that it's in our allowlist\n if k in self.allowlist:\n return self.globs[k]\n\n # somebody tried to be funny and do \"from sh import *\"\n if k == \"__all__\":\n warnings.warn(\n \"Cannot import * from sh. Please import sh or import programs \"\n \"individually.\"\n )\n return []\n\n # check if we're naming a dynamically generated ReturnCode exception\n exc = get_exc_from_name(k)\n if exc:\n return exc\n\n # https://github.com/ipython/ipython/issues/2577\n # https://github.com/amoffat/sh/issues/97#issuecomment-10610629\n if k.startswith(\"__\") and k.endswith(\"__\"):\n raise AttributeError\n\n # is it a command?\n cmd = resolve_command(k, self.globs[Command.__name__], self.baked_args)\n if cmd:\n return cmd\n\n # is it a custom builtin?\n builtin = getattr(self, \"b_\" + k, None)\n if builtin:\n return builtin\n\n # how about an environment variable?\n # this check must come after testing if its a command, because on some\n # systems, there are an environment variables that can conflict with\n # command names.\n # https://github.com/amoffat/sh/issues/238\n try:\n return os.environ[k]\n except KeyError:\n pass\n\n # nothing found, raise an exception\n raise CommandNotFound(k)\n\n # Methods that begin with \"b_\" are implementations of shell built-ins that\n # people are used to, but which may not have an executable equivalent.\n @staticmethod\n def b_which(program, paths=None):\n return _which(program, paths)\n\n\nclass Contrib(ModuleType): # pragma: no cover\n @classmethod\n def __call__(cls, name):\n def wrapper1(fn):\n @property\n def cmd_getter(self):\n cmd = resolve_command(name, Command)\n\n if not cmd:\n raise CommandNotFound(name)\n\n new_cmd = fn(cmd)\n return new_cmd\n\n setattr(cls, name, cmd_getter)\n return fn\n\n return wrapper1\n\n\nmod_name = __name__ + \".contrib\"\ncontrib = Contrib(mod_name)\nsys.modules[mod_name] = contrib\n\n\n@contrib(\"git\")\ndef git(orig): # pragma: no cover\n \"\"\"most git commands play nicer without a TTY\"\"\"\n cmd = orig.bake(_tty_out=False)\n return cmd\n\n\n@contrib(\"bash\")\ndef bash(orig):\n cmd = orig.bake(\"-c\")\n return cmd\n\n\n@contrib(\"sudo\")\ndef sudo(orig): # pragma: no cover\n \"\"\"a nicer version of sudo that uses getpass to ask for a password, or\n allows the first argument to be a string password\"\"\"\n\n prompt = f\"[sudo] password for {getpass.getuser()}: \"\n\n def stdin():\n pw = getpass.getpass(prompt=prompt) + \"\\n\"\n yield pw\n\n def process(a, kwargs):\n password = kwargs.pop(\"password\", None)\n\n if password is None:\n pass_getter = stdin()\n else:\n pass_getter = password.rstrip(\"\\n\") + \"\\n\"\n\n kwargs[\"_in\"] = pass_getter\n return a, kwargs\n\n cmd = orig.bake(\"-S\", _arg_preprocess=process)\n return cmd\n\n\n@contrib(\"ssh\")\ndef ssh(orig): # pragma: no cover\n \"\"\"An ssh command for automatic password login\"\"\"\n\n class SessionContent:\n def __init__(self):\n self.chars = deque(maxlen=50000)\n self.lines = deque(maxlen=5000)\n self.line_chars = []\n self.last_line = \"\"\n self.cur_char = \"\"\n\n def append_char(self, char):\n if char == \"\\n\":\n line = self.cur_line\n self.last_line = line\n self.lines.append(line)\n self.line_chars = []\n else:\n self.line_chars.append(char)\n\n self.chars.append(char)\n self.cur_char = char\n\n @property\n def cur_line(self):\n line = \"\".join(self.line_chars)\n return line\n\n class SSHInteract:\n def __init__(self, prompt_match, pass_getter, out_handler, login_success):\n self.prompt_match = prompt_match\n self.pass_getter = pass_getter\n self.out_handler = out_handler\n self.login_success = login_success\n self.content = SessionContent()\n\n # some basic state\n self.pw_entered = False\n self.success = False\n\n def __call__(self, char, stdin):\n self.content.append_char(char)\n\n if self.pw_entered and not self.success:\n self.success = self.login_success(self.content)\n\n if self.success:\n return self.out_handler(self.content, stdin)\n\n if self.prompt_match(self.content):\n password = self.pass_getter()\n stdin.put(password + \"\\n\")\n self.pw_entered = True\n\n def process(a, kwargs):\n real_out_handler = kwargs.pop(\"interact\")\n password = kwargs.pop(\"password\", None)\n login_success = kwargs.pop(\"login_success\", None)\n prompt_match = kwargs.pop(\"prompt\", None)\n prompt = \"Please enter SSH password: \"\n\n if prompt_match is None:\n\n def prompt_match(content):\n return content.cur_line.endswith(\"password: \") # noqa: E731\n\n if password is None:\n\n def pass_getter():\n return getpass.getpass(prompt=prompt) # noqa: E731\n\n else:\n\n def pass_getter():\n return password.rstrip(\"\\n\") # noqa: E731\n\n if login_success is None:\n\n def login_success(content):\n return True # noqa: E731\n\n kwargs[\"_out\"] = SSHInteract(\n prompt_match, pass_getter, real_out_handler, login_success\n )\n return a, kwargs\n\n cmd = orig.bake(\n _out_bufsize=0, _tty_in=True, _unify_ttys=True, _arg_preprocess=process\n )\n return cmd\n\n\ndef run_repl(env): # pragma: no cover\n print(f\"\\n>> sh v{__version__}\\n>> https://github.com/amoffat/sh\\n\")\n\n while True:\n try:\n line = input(\"sh> \")\n except (ValueError, EOFError):\n break\n\n try:\n exec(compile(line, \"\", \"single\"), env, env)\n except SystemExit:\n break\n except: # noqa: E722\n print(traceback.format_exc())\n\n # cleans up our last line\n print(\"\")\n\n\n# this is a thin wrapper around THIS module (we patch sys.modules[__name__]).\n# this is in the case that the user does a \"from sh import whatever\"\n# in other words, they only want to import certain programs, not the whole\n# system PATH worth of commands. in this case, we just proxy the\n# import lookup to our Environment class\nclass SelfWrapper(ModuleType):\n def __init__(self, self_module, baked_args=None):\n # this is super ugly to have to copy attributes like this,\n # but it seems to be the only way to make reload() behave\n # nicely. if i make these attributes dynamic lookups in\n # __getattr__, reload sometimes chokes in weird ways...\n super().__init__(\n name=getattr(self_module, \"__name__\", None),\n doc=getattr(self_module, \"__doc__\", None),\n )\n for attr in [\"__builtins__\", \"__file__\", \"__package__\"]:\n setattr(self, attr, getattr(self_module, attr, None))\n\n # python 3.2 (2.7 and 3.3 work fine) breaks on osx (not ubuntu)\n # if we set this to None. and 3.3 needs a value for __path__\n self.__path__ = []\n self.__self_module = self_module\n\n # Copy the Command class and add any baked call kwargs to it\n command_cls = Command\n cls_attrs = command_cls.__dict__.copy()\n cls_attrs.pop(\"__dict__\", None)\n if baked_args:\n call_args, _ = command_cls._extract_call_args(baked_args)\n cls_attrs[\"_call_args\"] = cls_attrs[\"_call_args\"].copy()\n cls_attrs[\"_call_args\"].update(call_args)\n globs = globals().copy()\n globs[command_cls.__name__] = type(\n command_cls.__name__, command_cls.__bases__, cls_attrs\n )\n\n self.__env = Environment(globs, baked_args=baked_args)\n\n def __getattr__(self, name):\n return self.__env[name]\n\n def bake(self, **kwargs):\n baked_args = self.__env.baked_args.copy()\n baked_args.update(kwargs)\n new_sh = self.__class__(self.__self_module, baked_args)\n return new_sh\n\n\nif __name__ == \"__main__\": # pragma: no cover\n # we're being run as a stand-alone script\n env = Environment(globals())\n run_repl(env)\nelse:\n # we're being imported from somewhere\n sys.modules[__name__] = SelfWrapper(sys.modules[__name__])\n" }, { "path": "tests/Dockerfile", "content": "FROM ubuntu:focal\n\nARG cache_bust\nRUN apt update &&\\\n apt -y install locales\n\nRUN locale-gen en_US.UTF-8\nENV LANG en_US.UTF-8\nENV LANGUAGE en_US:en\nENV LC_ALL en_US.UTF-8\nENV TZ Etc/UTC\nENV DEBIAN_FRONTEND noninteractive\n\nRUN apt -y install\\\n software-properties-common\\\n curl\\\n sudo\\\n lsof\n\nRUN add-apt-repository -y ppa:deadsnakes/ppa\nRUN apt update\nRUN apt -y install\\\n python3.8\\\n python3.9\\\n python3.10\\\n python3.11\n\nRUN apt -y install\\\n python3.8-distutils\\\n python3.9-distutils\\\n && curl https://bootstrap.pypa.io/get-pip.py | python3.9 -\n\nARG uid=1000\nRUN groupadd -g $uid shtest\\\n && useradd -m -u $uid -g $uid shtest\\\n && gpasswd -a shtest sudo\\\n && echo \"shtest:shtest\" | chpasswd\n\n\nENV TOX_PARALLEL_NO_SPINNER=1\nUSER shtest\nWORKDIR /home/shtest/\n\nENV PATH=\"/home/shtest/.local/bin:$PATH\"\nRUN pip install tox flake8 black rstcheck mypy\n\nCOPY README.rst sh.py .flake8 tox.ini tests/sh_test.py /home/shtest/\n" }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/sh_test.py", "content": "import asyncio\nimport errno\nimport fcntl\nimport inspect\nimport logging\nimport os\nimport platform\nimport pty\nimport resource\nimport signal\nimport stat\nimport sys\nimport tempfile\nimport time\nimport unittest\nimport unittest.mock\nimport warnings\nfrom asyncio.queues import Queue as AQueue\nfrom contextlib import contextmanager\nfrom functools import partial, wraps\nfrom hashlib import md5\nfrom io import BytesIO, StringIO\nfrom os.path import dirname, exists, join, realpath, split\nfrom pathlib import Path\n\nimport sh\n\nTHIS_DIR = Path(__file__).resolve().parent\nRAND_BYTES = os.urandom(10)\n\n# we have to use the real path because on osx, /tmp is a symlink to\n# /private/tmp, and so assertions that gettempdir() == sh.pwd() will fail\ntempdir = Path(tempfile.gettempdir()).resolve()\nIS_MACOS = platform.system() in (\"AIX\", \"Darwin\")\n\nSIGNALS_THAT_SHOULD_THROW_EXCEPTION = [\n signal.SIGABRT,\n signal.SIGBUS,\n signal.SIGFPE,\n signal.SIGILL,\n signal.SIGINT,\n signal.SIGKILL,\n signal.SIGPIPE,\n signal.SIGQUIT,\n signal.SIGSEGV,\n signal.SIGTERM,\n signal.SIGSYS,\n]\n\n\ndef hash(a: str):\n h = md5(a.encode(\"utf8\") + RAND_BYTES)\n return h.hexdigest()\n\n\ndef randomize_order(a, b):\n h1 = hash(a)\n h2 = hash(b)\n if h1 == h2:\n return 0\n elif h1 < h2:\n return -1\n else:\n return 1\n\n\nunittest.TestLoader.sortTestMethodsUsing = staticmethod(randomize_order)\n\n\n# these 3 functions are helpers for modifying PYTHONPATH with a module's main\n# directory\n\n\ndef append_pythonpath(env, path):\n key = \"PYTHONPATH\"\n pypath = [p for p in env.get(key, \"\").split(\":\") if p]\n pypath.insert(0, path)\n pypath = \":\".join(pypath)\n env[key] = pypath\n\n\ndef get_module_import_dir(m):\n mod_file = inspect.getsourcefile(m)\n is_package = mod_file.endswith(\"__init__.py\")\n\n mod_dir = dirname(mod_file)\n if is_package:\n mod_dir, _ = split(mod_dir)\n return mod_dir\n\n\ndef append_module_path(env, m):\n append_pythonpath(env, get_module_import_dir(m))\n\n\nsystem_python = sh.Command(sys.executable)\n\n# this is to ensure that our `python` helper here is able to import our local sh\n# module, and not the system one\nbaked_env = os.environ.copy()\nappend_module_path(baked_env, sh)\npython = system_python.bake(_env=baked_env, _return_cmd=True)\npythons = python.bake(_return_cmd=False)\npython_bg = system_python.bake(_env=baked_env, _bg=True)\n\n\ndef requires_progs(*progs):\n missing = []\n for prog in progs:\n try:\n sh.Command(prog)\n except sh.CommandNotFound:\n missing.append(prog)\n\n friendly_missing = \", \".join(missing)\n return unittest.skipUnless(\n len(missing) == 0, f\"Missing required system programs: {friendly_missing}\"\n )\n\n\nrequires_posix = unittest.skipUnless(os.name == \"posix\", \"Requires POSIX\")\nrequires_utf8 = unittest.skipUnless(\n sh.DEFAULT_ENCODING == \"UTF-8\", \"System encoding must be UTF-8\"\n)\nnot_macos = unittest.skipUnless(not IS_MACOS, \"Doesn't work on MacOS\")\n\n\ndef requires_poller(poller):\n use_select = bool(int(os.environ.get(\"SH_TESTS_USE_SELECT\", \"0\")))\n cur_poller = \"select\" if use_select else \"poll\"\n return unittest.skipUnless(\n cur_poller == poller, f\"Only enabled for select.{cur_poller}\"\n )\n\n\n@contextmanager\ndef ulimit(key, new_soft):\n soft, hard = resource.getrlimit(key)\n resource.setrlimit(key, (new_soft, hard))\n try:\n yield\n finally:\n resource.setrlimit(key, (soft, hard))\n\n\ndef create_tmp_test(code, prefix=\"tmp\", delete=True, **kwargs):\n \"\"\"creates a temporary test file that lives on disk, on which we can run\n python with sh\"\"\"\n\n py = tempfile.NamedTemporaryFile(prefix=prefix, delete=delete)\n\n code = code.format(**kwargs)\n code = code.encode(\"UTF-8\")\n\n py.write(code)\n py.flush()\n\n # make the file executable\n st = os.stat(py.name)\n os.chmod(py.name, st.st_mode | stat.S_IEXEC)\n\n # we don't explicitly close, because close will remove the file, and we\n # don't want that until the test case is done. so we let the gc close it\n # when it goes out of scope\n return py\n\n\nclass BaseTests(unittest.TestCase):\n def setUp(self):\n warnings.simplefilter(\"ignore\", ResourceWarning)\n\n def tearDown(self):\n warnings.simplefilter(\"default\", ResourceWarning)\n\n def assert_oserror(self, num, fn, *args, **kwargs):\n try:\n fn(*args, **kwargs)\n except OSError as e:\n self.assertEqual(e.errno, num)\n\n def assert_deprecated(self, fn, *args, **kwargs):\n with warnings.catch_warnings(record=True) as w:\n fn(*args, **kwargs)\n\n self.assertEqual(len(w), 1)\n self.assertTrue(issubclass(w[-1].category, DeprecationWarning))\n\n\nclass ArgTests(BaseTests):\n def test_list_args(self):\n processed = sh._aggregate_keywords({\"arg\": [1, 2, 3]}, \"=\", \"--\")\n self.assertListEqual(processed, [\"--arg=1\", \"--arg=2\", \"--arg=3\"])\n\n def test_bool_values(self):\n processed = sh._aggregate_keywords({\"truthy\": True, \"falsey\": False}, \"=\", \"--\")\n self.assertListEqual(processed, [\"--truthy\"])\n\n def test_space_sep(self):\n processed = sh._aggregate_keywords({\"arg\": \"123\"}, \" \", \"--\")\n self.assertListEqual(processed, [\"--arg\", \"123\"])\n\n\n@requires_posix\nclass FunctionalTests(BaseTests):\n def setUp(self):\n self._environ = os.environ.copy()\n super().setUp()\n\n def tearDown(self):\n os.environ = self._environ\n super().tearDown()\n\n def test_print_command(self):\n from sh import ls, which\n\n actual_location = which(\"ls\").strip()\n out = str(ls)\n self.assertEqual(out, actual_location)\n\n def test_unicode_arg(self):\n from sh import echo\n\n test = \"漢字\"\n p = echo(test, _encoding=\"utf8\")\n output = p.strip()\n self.assertEqual(test, output)\n\n def test_unicode_exception(self):\n from sh import ErrorReturnCode\n\n py = create_tmp_test(\"exit(1)\")\n\n arg = \"漢字\"\n native_arg = arg\n\n try:\n python(py.name, arg, _encoding=\"utf8\")\n except ErrorReturnCode as e:\n self.assertIn(native_arg, str(e))\n else:\n self.fail(\"exception wasn't raised\")\n\n def test_pipe_fd(self):\n py = create_tmp_test(\"\"\"print(\"hi world\")\"\"\")\n read_fd, write_fd = os.pipe()\n python(py.name, _out=write_fd)\n out = os.read(read_fd, 10)\n self.assertEqual(out, b\"hi world\\n\")\n\n def test_trunc_exc(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(\"a\" * 1000)\nsys.stderr.write(\"b\" * 1000)\nexit(1)\n\"\"\"\n )\n self.assertRaises(sh.ErrorReturnCode_1, python, py.name)\n\n def test_number_arg(self):\n py = create_tmp_test(\n \"\"\"\nfrom optparse import OptionParser\nparser = OptionParser()\noptions, args = parser.parse_args()\nprint(args[0])\n\"\"\"\n )\n\n out = python(py.name, 3).strip()\n self.assertEqual(out, \"3\")\n\n def test_arg_string_coercion(self):\n py = create_tmp_test(\n \"\"\"\nfrom argparse import ArgumentParser\nparser = ArgumentParser()\nparser.add_argument(\"-n\", type=int)\nparser.add_argument(\"--number\", type=int)\nns = parser.parse_args()\nprint(ns.n + ns.number)\n\"\"\"\n )\n\n out = python(py.name, n=3, number=4, _long_sep=None).strip()\n self.assertEqual(out, \"7\")\n\n def test_empty_stdin_no_hang(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\ndata = sys.stdin.read()\nsys.stdout.write(\"no hang\")\n\"\"\"\n )\n out = pythons(py.name, _in=\"\", _timeout=2)\n self.assertEqual(out, \"no hang\")\n\n out = pythons(py.name, _in=None, _timeout=2)\n self.assertEqual(out, \"no hang\")\n\n def test_exit_code(self):\n from sh import ErrorReturnCode_3\n\n py = create_tmp_test(\n \"\"\"\nexit(3)\n\"\"\"\n )\n self.assertRaises(ErrorReturnCode_3, python, py.name)\n\n def test_patched_glob(self):\n from glob import glob\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nprint(sys.argv[1:])\n\"\"\"\n )\n files = glob(\"*.faowjefoajweofj\")\n out = python(py.name, files).strip()\n self.assertEqual(out, \"['*.faowjefoajweofj']\")\n\n def test_exit_code_with_hasattr(self):\n from sh import ErrorReturnCode_3\n\n py = create_tmp_test(\n \"\"\"\nexit(3)\n\"\"\"\n )\n\n try:\n out = python(py.name, _iter=True)\n # hasattr can swallow exceptions\n hasattr(out, \"something_not_there\")\n list(out)\n self.assertEqual(out.exit_code, 3)\n self.fail(\"Command exited with error, but no exception thrown\")\n except ErrorReturnCode_3:\n pass\n\n def test_exit_code_from_exception(self):\n from sh import ErrorReturnCode_3\n\n py = create_tmp_test(\n \"\"\"\nexit(3)\n\"\"\"\n )\n\n self.assertRaises(ErrorReturnCode_3, python, py.name)\n\n try:\n python(py.name)\n except Exception as e:\n self.assertEqual(e.exit_code, 3)\n\n def test_stdin_from_string(self):\n from sh import sed\n\n self.assertEqual(\n sed(_in=\"one test three\", e=\"s/test/two/\").strip(), \"one two three\"\n )\n\n def test_ok_code(self):\n from sh import ErrorReturnCode_1, ErrorReturnCode_2, ls\n\n exc_to_test = ErrorReturnCode_2\n code_to_pass = 2\n if IS_MACOS:\n exc_to_test = ErrorReturnCode_1\n code_to_pass = 1\n self.assertRaises(exc_to_test, ls, \"/aofwje/garogjao4a/eoan3on\")\n\n ls(\"/aofwje/garogjao4a/eoan3on\", _ok_code=code_to_pass)\n ls(\"/aofwje/garogjao4a/eoan3on\", _ok_code=[code_to_pass])\n ls(\"/aofwje/garogjao4a/eoan3on\", _ok_code=range(code_to_pass + 1))\n\n def test_ok_code_none(self):\n py = create_tmp_test(\"exit(0)\")\n python(py.name, _ok_code=None)\n\n def test_ok_code_exception(self):\n from sh import ErrorReturnCode_0\n\n py = create_tmp_test(\"exit(0)\")\n self.assertRaises(ErrorReturnCode_0, python, py.name, _ok_code=2)\n\n def test_none_arg(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nprint(sys.argv[1:])\n\"\"\"\n )\n maybe_arg = \"some\"\n out = python(py.name, maybe_arg).strip()\n self.assertEqual(out, \"['some']\")\n\n maybe_arg = None\n out = python(py.name, maybe_arg).strip()\n self.assertEqual(out, \"[]\")\n\n def test_quote_escaping(self):\n py = create_tmp_test(\n \"\"\"\nfrom optparse import OptionParser\nparser = OptionParser()\noptions, args = parser.parse_args()\nprint(args)\n\"\"\"\n )\n out = python(py.name, \"one two three\").strip()\n self.assertEqual(out, \"['one two three']\")\n\n out = python(py.name, 'one \"two three').strip()\n self.assertEqual(out, \"['one \\\"two three']\")\n\n out = python(py.name, \"one\", \"two three\").strip()\n self.assertEqual(out, \"['one', 'two three']\")\n\n out = python(py.name, \"one\", 'two \"haha\" three').strip()\n self.assertEqual(out, \"['one', 'two \\\"haha\\\" three']\")\n\n out = python(py.name, \"one two's three\").strip()\n self.assertEqual(out, '[\"one two\\'s three\"]')\n\n out = python(py.name, \"one two's three\").strip()\n self.assertEqual(out, '[\"one two\\'s three\"]')\n\n def test_multiple_pipes(self):\n import time\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\n\nfor l in \"andrew\":\n sys.stdout.write(l)\n time.sleep(.2)\n\"\"\"\n )\n\n inc_py = create_tmp_test(\n \"\"\"\nimport sys\nwhile True:\n letter = sys.stdin.read(1)\n if not letter:\n break\n sys.stdout.write(chr(ord(letter)+1))\n\"\"\"\n )\n\n def inc(*args, **kwargs):\n return python(\"-u\", inc_py.name, *args, **kwargs)\n\n class Derp:\n def __init__(self):\n self.times = []\n self.stdout = []\n self.last_received = None\n\n def agg(self, line):\n self.stdout.append(line.strip())\n now = time.time()\n if self.last_received:\n self.times.append(now - self.last_received)\n self.last_received = now\n\n derp = Derp()\n\n p = inc(\n _in=inc(\n _in=inc(_in=python(\"-u\", py.name, _piped=True), _piped=True),\n _piped=True,\n ),\n _out=derp.agg,\n )\n\n p.wait()\n self.assertEqual(\"\".join(derp.stdout), \"dqguhz\")\n self.assertTrue(all([t > 0.15 for t in derp.times]))\n\n def test_manual_stdin_string(self):\n from sh import tr\n\n out = tr(\"[:lower:]\", \"[:upper:]\", _in=\"andrew\").strip()\n self.assertEqual(out, \"ANDREW\")\n\n def test_manual_stdin_iterable(self):\n from sh import tr\n\n test = [\"testing\\n\", \"herp\\n\", \"derp\\n\"]\n out = tr(\"[:lower:]\", \"[:upper:]\", _in=test)\n\n match = \"\".join([t.upper() for t in test])\n self.assertEqual(out, match)\n\n def test_manual_stdin_file(self):\n import tempfile\n\n from sh import tr\n\n test_string = \"testing\\nherp\\nderp\\n\"\n\n stdin = tempfile.NamedTemporaryFile()\n stdin.write(test_string.encode())\n stdin.flush()\n stdin.seek(0)\n\n out = tr(\"[:lower:]\", \"[:upper:]\", _in=stdin)\n\n self.assertEqual(out, test_string.upper())\n\n def test_manual_stdin_queue(self):\n from sh import tr\n\n try:\n from Queue import Queue\n except ImportError:\n from queue import Queue\n\n test = [\"testing\\n\", \"herp\\n\", \"derp\\n\"]\n\n q = Queue()\n for t in test:\n q.put(t)\n q.put(None) # EOF\n\n out = tr(\"[:lower:]\", \"[:upper:]\", _in=q)\n\n match = \"\".join([t.upper() for t in test])\n self.assertEqual(out, match)\n\n def test_environment(self):\n \"\"\"tests that environments variables that we pass into sh commands\n exist in the environment, and on the sh module\"\"\"\n import os\n\n # this is the environment we'll pass into our commands\n env = {\"HERP\": \"DERP\"}\n\n # first we test that the environment exists in our child process as\n # we've set it\n py = create_tmp_test(\n \"\"\"\nimport os\n\nfor key in list(os.environ.keys()):\n if key != \"HERP\":\n del os.environ[key]\nprint(dict(os.environ))\n\"\"\"\n )\n out = python(py.name, _env=env).strip()\n self.assertEqual(out, \"{'HERP': 'DERP'}\")\n\n py = create_tmp_test(\n \"\"\"\nimport os, sys\nsys.path.insert(0, os.getcwd())\nimport sh\nfor key in list(os.environ.keys()):\n if key != \"HERP\":\n del os.environ[key]\nprint(dict(HERP=sh.HERP))\n\"\"\"\n )\n out = python(py.name, _env=env, _cwd=THIS_DIR).strip()\n self.assertEqual(out, \"{'HERP': 'DERP'}\")\n\n # Test that _env also accepts os.environ which is a mpping but not a dict.\n os.environ[\"HERP\"] = \"DERP\"\n out = python(py.name, _env=os.environ, _cwd=THIS_DIR).strip()\n self.assertEqual(out, \"{'HERP': 'DERP'}\")\n\n def test_which(self):\n # Test 'which' as built-in function\n from sh import ls\n\n which = sh._SelfWrapper__env.b_which\n self.assertEqual(which(\"fjoawjefojawe\"), None)\n self.assertEqual(which(\"ls\"), str(ls))\n\n def test_which_paths(self):\n # Test 'which' as built-in function\n which = sh._SelfWrapper__env.b_which\n py = create_tmp_test(\n \"\"\"\nprint(\"hi\")\n\"\"\"\n )\n test_path = dirname(py.name)\n _, test_name = os.path.split(py.name)\n\n found_path = which(test_name)\n self.assertEqual(found_path, None)\n\n found_path = which(test_name, [test_path])\n self.assertEqual(found_path, py.name)\n\n def test_no_close_fds(self):\n # guarantee some extra fds in our parent process that don't close on exec. we\n # have to explicitly do this because at some point (I believe python 3.4),\n # python started being more stringent with closing fds to prevent security\n # vulnerabilities. python 2.7, for example, doesn't set CLOEXEC on\n # tempfile.TemporaryFile()s\n #\n # https://www.python.org/dev/peps/pep-0446/\n tmp = [tempfile.TemporaryFile() for i in range(10)]\n for t in tmp:\n flags = fcntl.fcntl(t.fileno(), fcntl.F_GETFD)\n flags &= ~fcntl.FD_CLOEXEC\n fcntl.fcntl(t.fileno(), fcntl.F_SETFD, flags)\n\n py = create_tmp_test(\n \"\"\"\nimport os\nprint(len(os.listdir(\"/dev/fd\")))\n\"\"\"\n )\n out = python(py.name, _close_fds=False).strip()\n # pick some number greater than 4, since it's hard to know exactly how many fds\n # will be open/inherted in the child\n self.assertGreater(int(out), 7)\n\n for t in tmp:\n t.close()\n\n def test_close_fds(self):\n # guarantee some extra fds in our parent process that don't close on exec.\n # we have to explicitly do this because at some point (I believe python 3.4),\n # python started being more stringent with closing fds to prevent security\n # vulnerabilities. python 2.7, for example, doesn't set CLOEXEC on\n # tempfile.TemporaryFile()s\n #\n # https://www.python.org/dev/peps/pep-0446/\n tmp = [tempfile.TemporaryFile() for i in range(10)]\n for t in tmp:\n flags = fcntl.fcntl(t.fileno(), fcntl.F_GETFD)\n flags &= ~fcntl.FD_CLOEXEC\n fcntl.fcntl(t.fileno(), fcntl.F_SETFD, flags)\n\n py = create_tmp_test(\n \"\"\"\nimport os\nprint(os.listdir(\"/dev/fd\"))\n\"\"\"\n )\n out = python(py.name).strip()\n self.assertEqual(out, \"['0', '1', '2', '3']\")\n\n for t in tmp:\n t.close()\n\n def test_pass_fds(self):\n # guarantee some extra fds in our parent process that don't close on exec.\n # we have to explicitly do this because at some point (I believe python 3.4),\n # python started being more stringent with closing fds to prevent security\n # vulnerabilities. python 2.7, for example, doesn't set CLOEXEC on\n # tempfile.TemporaryFile()s\n #\n # https://www.python.org/dev/peps/pep-0446/\n tmp = [tempfile.TemporaryFile() for i in range(10)]\n for t in tmp:\n flags = fcntl.fcntl(t.fileno(), fcntl.F_GETFD)\n flags &= ~fcntl.FD_CLOEXEC\n fcntl.fcntl(t.fileno(), fcntl.F_SETFD, flags)\n last_fd = tmp[-1].fileno()\n\n py = create_tmp_test(\n \"\"\"\nimport os\nprint(os.listdir(\"/dev/fd\"))\n\"\"\"\n )\n out = python(py.name, _pass_fds=[last_fd]).strip()\n inherited = [0, 1, 2, 3, last_fd]\n inherited_str = [str(i) for i in inherited]\n self.assertEqual(out, str(inherited_str))\n\n for t in tmp:\n t.close()\n\n def test_no_arg(self):\n import pwd\n\n from sh import whoami\n\n u1 = whoami().strip()\n u2 = pwd.getpwuid(os.geteuid())[0]\n self.assertEqual(u1, u2)\n\n def test_incompatible_special_args(self):\n from sh import ls\n\n self.assertRaises(TypeError, ls, _iter=True, _piped=True)\n\n def test_invalid_env(self):\n from sh import ls\n\n exc = TypeError\n self.assertRaises(exc, ls, _env=\"XXX\")\n self.assertRaises(exc, ls, _env={\"foo\": 123})\n self.assertRaises(exc, ls, _env={123: \"bar\"})\n\n def test_exception(self):\n from sh import ErrorReturnCode_2\n\n py = create_tmp_test(\n \"\"\"\nexit(2)\n\"\"\"\n )\n self.assertRaises(ErrorReturnCode_2, python, py.name)\n\n def test_piped_exception1(self):\n from sh import ErrorReturnCode_2\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(\"line1\\\\n\")\nsys.stdout.write(\"line2\\\\n\")\nsys.stdout.flush()\nexit(2)\n\"\"\"\n )\n\n py2 = create_tmp_test(\"\")\n\n def fn():\n list(python(python(py.name, _piped=True), \"-u\", py2.name, _iter=True))\n\n self.assertRaises(ErrorReturnCode_2, fn)\n\n def test_piped_exception2(self):\n from sh import ErrorReturnCode_2\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(\"line1\\\\n\")\nsys.stdout.write(\"line2\\\\n\")\nsys.stdout.flush()\nexit(2)\n\"\"\"\n )\n\n py2 = create_tmp_test(\"\")\n\n def fn():\n python(python(py.name, _piped=True), \"-u\", py2.name)\n\n self.assertRaises(ErrorReturnCode_2, fn)\n\n def test_command_not_found(self):\n from sh import CommandNotFound\n\n def do_import():\n from sh import aowjgoawjoeijaowjellll # noqa: F401\n\n self.assertRaises(ImportError, do_import)\n\n def do_import():\n import sh\n\n sh.awoefaowejfw\n\n self.assertRaises(CommandNotFound, do_import)\n\n def do_import():\n import sh\n\n sh.Command(\"ofajweofjawoe\")\n\n self.assertRaises(CommandNotFound, do_import)\n\n def test_command_wrapper_equivalence(self):\n from sh import Command, ls, which\n\n self.assertEqual(Command(str(which(\"ls\")).strip()), ls)\n\n def test_doesnt_execute_directories(self):\n save_path = os.environ[\"PATH\"]\n bin_dir1 = tempfile.mkdtemp()\n bin_dir2 = tempfile.mkdtemp()\n gcc_dir1 = os.path.join(bin_dir1, \"gcc\")\n gcc_file2 = os.path.join(bin_dir2, \"gcc\")\n try:\n os.environ[\"PATH\"] = os.pathsep.join((bin_dir1, bin_dir2))\n # a folder named 'gcc', its executable, but should not be\n # discovered by internal which(1)-clone\n os.makedirs(gcc_dir1)\n # an executable named gcc -- only this should be executed\n bunk_header = \"#!/bin/sh\\necho $*\"\n with open(gcc_file2, \"w\") as h:\n h.write(bunk_header)\n os.chmod(gcc_file2, int(0o755))\n\n from sh import gcc\n\n self.assertEqual(gcc._path, gcc_file2)\n self.assertEqual(\n gcc(\"no-error\", _return_cmd=True).stdout.strip(),\n b\"no-error\",\n )\n\n finally:\n os.environ[\"PATH\"] = save_path\n if exists(gcc_file2):\n os.unlink(gcc_file2)\n if exists(gcc_dir1):\n os.rmdir(gcc_dir1)\n if exists(bin_dir1):\n os.rmdir(bin_dir1)\n if exists(bin_dir1):\n os.rmdir(bin_dir2)\n\n def test_multiple_args_short_option(self):\n py = create_tmp_test(\n \"\"\"\nfrom optparse import OptionParser\nparser = OptionParser()\nparser.add_option(\"-l\", dest=\"long_option\")\noptions, args = parser.parse_args()\nprint(len(options.long_option.split()))\n\"\"\"\n )\n num_args = int(python(py.name, l=\"one two three\")) # noqa: E741\n self.assertEqual(num_args, 3)\n\n num_args = int(python(py.name, \"-l\", \"one's two's three's\"))\n self.assertEqual(num_args, 3)\n\n def test_multiple_args_long_option(self):\n py = create_tmp_test(\n \"\"\"\nfrom optparse import OptionParser\nparser = OptionParser()\nparser.add_option(\"-l\", \"--long-option\", dest=\"long_option\")\noptions, args = parser.parse_args()\nprint(len(options.long_option.split()))\n\"\"\"\n )\n num_args = int(python(py.name, long_option=\"one two three\", nothing=False))\n self.assertEqual(num_args, 3)\n\n num_args = int(python(py.name, \"--long-option\", \"one's two's three's\"))\n self.assertEqual(num_args, 3)\n\n num_args = int(python(py.name, long_option=\"one's two's three's\"))\n self.assertEqual(num_args, 3)\n\n cmd = str(python.bake(py.name, long_option=\"one two three\"))\n self.assertTrue(cmd.endswith(\" '--long-option=one two three'\"), cmd)\n\n def test_short_bool_option(self):\n py = create_tmp_test(\n \"\"\"\nfrom optparse import OptionParser\nparser = OptionParser()\nparser.add_option(\"-s\", action=\"store_true\", default=False, dest=\"short_option\")\noptions, args = parser.parse_args()\nprint(options.short_option)\n\"\"\"\n )\n self.assertTrue(python(py.name, s=True).strip() == \"True\")\n self.assertTrue(python(py.name, s=False).strip() == \"False\")\n self.assertTrue(python(py.name).strip() == \"False\")\n\n def test_long_bool_option(self):\n py = create_tmp_test(\n \"\"\"\nfrom optparse import OptionParser\nparser = OptionParser()\nparser.add_option(\"-l\", \"--long-option\", action=\"store_true\", default=False, \\\n dest=\"long_option\")\noptions, args = parser.parse_args()\nprint(options.long_option)\n\"\"\"\n )\n self.assertTrue(python(py.name, long_option=True).strip() == \"True\")\n self.assertTrue(python(py.name).strip() == \"False\")\n\n def test_false_bool_ignore(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nprint(sys.argv[1:])\n\"\"\"\n )\n test = True\n self.assertEqual(python(py.name, test and \"-n\").strip(), \"['-n']\")\n test = False\n self.assertEqual(python(py.name, test and \"-n\").strip(), \"[]\")\n\n def test_composition(self):\n py1 = create_tmp_test(\n \"\"\"\nimport sys\nprint(int(sys.argv[1]) * 2)\n \"\"\"\n )\n\n py2 = create_tmp_test(\n \"\"\"\nimport sys\nprint(int(sys.argv[1]) + 1)\n \"\"\"\n )\n\n res = python(py2.name, python(py1.name, 8)).strip()\n self.assertEqual(\"17\", res)\n\n def test_incremental_composition(self):\n py1 = create_tmp_test(\n \"\"\"\nimport sys\nprint(int(sys.argv[1]) * 2)\n \"\"\"\n )\n\n py2 = create_tmp_test(\n \"\"\"\nimport sys\nprint(int(sys.stdin.read()) + 1)\n \"\"\"\n )\n\n res = python(py2.name, _in=python(py1.name, 8, _piped=True)).strip()\n self.assertEqual(\"17\", res)\n\n def test_short_option(self):\n from sh import sh\n\n s1 = sh(c=\"echo test\").strip()\n s2 = \"test\"\n self.assertEqual(s1, s2)\n\n def test_long_option(self):\n py = create_tmp_test(\n \"\"\"\nfrom optparse import OptionParser\nparser = OptionParser()\nparser.add_option(\"-l\", \"--long-option\", action=\"store\", default=\"\", dest=\"long_option\")\noptions, args = parser.parse_args()\nprint(options.long_option.upper())\n\"\"\"\n )\n self.assertTrue(python(py.name, long_option=\"testing\").strip() == \"TESTING\")\n self.assertTrue(python(py.name).strip() == \"\")\n\n def test_raw_args(self):\n py = create_tmp_test(\n \"\"\"\nfrom optparse import OptionParser\nparser = OptionParser()\nparser.add_option(\"--long_option\", action=\"store\", default=None,\n dest=\"long_option1\")\nparser.add_option(\"--long-option\", action=\"store\", default=None,\n dest=\"long_option2\")\noptions, args = parser.parse_args()\n\nif options.long_option1:\n print(options.long_option1.upper())\nelse:\n print(options.long_option2.upper())\n\"\"\"\n )\n self.assertEqual(\n python(py.name, {\"long_option\": \"underscore\"}).strip(), \"UNDERSCORE\"\n )\n\n self.assertEqual(python(py.name, long_option=\"hyphen\").strip(), \"HYPHEN\")\n\n def test_custom_separator(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nprint(sys.argv[1])\n\"\"\"\n )\n\n opt = {\"long-option\": \"underscore\"}\n correct = \"--long-option=custom=underscore\"\n\n out = python(py.name, opt, _long_sep=\"=custom=\").strip()\n self.assertEqual(out, correct)\n\n # test baking too\n correct = \"--long-option=baked=underscore\"\n python_baked = python.bake(py.name, opt, _long_sep=\"=baked=\")\n out = python_baked().strip()\n self.assertEqual(out, correct)\n\n def test_custom_separator_space(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nprint(str(sys.argv[1:]))\n\"\"\"\n )\n opt = {\"long-option\": \"space\"}\n correct = [\"--long-option\", \"space\"]\n out = python(py.name, opt, _long_sep=\" \").strip()\n self.assertEqual(out, str(correct))\n\n def test_custom_long_prefix(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nprint(sys.argv[1])\n\"\"\"\n )\n\n out = python(\n py.name, {\"long-option\": \"underscore\"}, _long_prefix=\"-custom-\"\n ).strip()\n self.assertEqual(out, \"-custom-long-option=underscore\")\n\n out = python(py.name, {\"long-option\": True}, _long_prefix=\"-custom-\").strip()\n self.assertEqual(out, \"-custom-long-option\")\n\n # test baking too\n out = python.bake(\n py.name, {\"long-option\": \"underscore\"}, _long_prefix=\"-baked-\"\n )().strip()\n self.assertEqual(out, \"-baked-long-option=underscore\")\n\n out = python.bake(\n py.name, {\"long-option\": True}, _long_prefix=\"-baked-\"\n )().strip()\n self.assertEqual(out, \"-baked-long-option\")\n\n def test_command_wrapper(self):\n from sh import Command, which\n\n ls = Command(str(which(\"ls\")).strip())\n wc = Command(str(which(\"wc\")).strip())\n\n c1 = int(wc(l=True, _in=ls(\"-A1\", THIS_DIR, _return_cmd=True))) # noqa: E741\n c2 = len(os.listdir(THIS_DIR))\n\n self.assertEqual(c1, c2)\n\n def test_background(self):\n import time\n\n from sh import sleep\n\n start = time.time()\n sleep_time = 0.5\n p = sleep(sleep_time, _bg=True)\n\n now = time.time()\n self.assertLess(now - start, sleep_time)\n\n p.wait()\n now = time.time()\n self.assertGreater(now - start, sleep_time)\n\n def test_background_exception(self):\n py = create_tmp_test(\"exit(1)\")\n p = python(py.name, _bg=True, _bg_exc=False) # should not raise\n self.assertRaises(sh.ErrorReturnCode_1, p.wait) # should raise\n\n def test_with_context(self):\n import getpass\n\n from sh import whoami\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport subprocess\n\nprint(\"with_context\")\nsubprocess.Popen(sys.argv[1:], shell=False).wait()\n\"\"\"\n )\n\n cmd1 = python.bake(py.name, _with=True)\n with cmd1:\n out = whoami()\n self.assertIn(\"with_context\", out)\n self.assertIn(getpass.getuser(), out)\n\n def test_with_context_args(self):\n import getpass\n\n from sh import whoami\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport subprocess\nfrom optparse import OptionParser\n\nparser = OptionParser()\nparser.add_option(\"-o\", \"--opt\", action=\"store_true\", default=False, dest=\"opt\")\noptions, args = parser.parse_args()\n\nif options.opt:\n subprocess.Popen(args[0], shell=False).wait()\n\"\"\"\n )\n with python(py.name, opt=True, _with=True):\n out = whoami()\n self.assertEqual(getpass.getuser(), out.strip())\n\n with python(py.name, _with=True):\n out = whoami()\n self.assertEqual(out.strip(), \"\")\n\n def test_with_context_nested(self):\n echo_path = sh.echo._path\n with sh.echo.bake(\"test1\", _with=True):\n with sh.echo.bake(\"test2\", _with=True):\n out = sh.echo(\"test3\")\n self.assertEqual(out.strip(), f\"test1 {echo_path} test2 {echo_path} test3\")\n\n def test_binary_input(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\ndata = sys.stdin.read()\nsys.stdout.write(data)\n\"\"\"\n )\n data = b\"1234\"\n out = pythons(py.name, _in=data)\n self.assertEqual(out, \"1234\")\n\n def test_err_to_out(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nsys.stdout.write(\"stdout\")\nsys.stdout.flush()\nsys.stderr.write(\"stderr\")\nsys.stderr.flush()\n\"\"\"\n )\n stdout = pythons(py.name, _err_to_out=True)\n self.assertEqual(stdout, \"stdoutstderr\")\n\n def test_err_to_out_and_sys_stdout(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nsys.stdout.write(\"stdout\")\nsys.stdout.flush()\nsys.stderr.write(\"stderr\")\nsys.stderr.flush()\n\"\"\"\n )\n master, slave = os.pipe()\n stdout = pythons(py.name, _err_to_out=True, _out=slave)\n self.assertEqual(stdout, \"\")\n self.assertEqual(os.read(master, 12), b\"stdoutstderr\")\n\n def test_err_piped(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stderr.write(\"stderr\")\n\"\"\"\n )\n\n py2 = create_tmp_test(\n \"\"\"\nimport sys\nwhile True:\n line = sys.stdin.read()\n if not line:\n break\n sys.stdout.write(line)\n\"\"\"\n )\n\n out = pythons(\"-u\", py2.name, _in=python(\"-u\", py.name, _piped=\"err\"))\n self.assertEqual(out, \"stderr\")\n\n def test_out_redirection(self):\n import tempfile\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nsys.stdout.write(\"stdout\")\nsys.stderr.write(\"stderr\")\n\"\"\"\n )\n\n file_obj = tempfile.NamedTemporaryFile()\n out = python(py.name, _out=file_obj)\n\n self.assertEqual(len(out), 0)\n\n file_obj.seek(0)\n actual_out = file_obj.read()\n file_obj.close()\n\n self.assertNotEqual(len(actual_out), 0)\n\n # test with tee\n file_obj = tempfile.NamedTemporaryFile()\n out = python(py.name, _out=file_obj, _tee=True)\n\n self.assertGreater(len(out), 0)\n\n file_obj.seek(0)\n actual_out = file_obj.read()\n file_obj.close()\n\n self.assertGreater(len(actual_out), 0)\n\n def test_err_redirection(self):\n import tempfile\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nsys.stdout.write(\"stdout\")\nsys.stderr.write(\"stderr\")\n\"\"\"\n )\n file_obj = tempfile.NamedTemporaryFile()\n p = python(\"-u\", py.name, _err=file_obj)\n\n file_obj.seek(0)\n stderr = file_obj.read().decode()\n file_obj.close()\n\n self.assertEqual(p.stdout, b\"stdout\")\n self.assertEqual(stderr, \"stderr\")\n self.assertEqual(len(p.stderr), 0)\n\n # now with tee\n file_obj = tempfile.NamedTemporaryFile()\n p = python(py.name, _err=file_obj, _tee=\"err\")\n\n file_obj.seek(0)\n stderr = file_obj.read().decode()\n file_obj.close()\n\n self.assertEqual(p.stdout, b\"stdout\")\n self.assertEqual(stderr, \"stderr\")\n self.assertGreater(len(p.stderr), 0)\n\n def test_out_and_err_redirection(self):\n import tempfile\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nsys.stdout.write(\"stdout\")\nsys.stderr.write(\"stderr\")\n\"\"\"\n )\n err_file_obj = tempfile.NamedTemporaryFile()\n out_file_obj = tempfile.NamedTemporaryFile()\n p = python(py.name, _out=out_file_obj, _err=err_file_obj, _tee=(\"err\", \"out\"))\n\n out_file_obj.seek(0)\n stdout = out_file_obj.read().decode()\n out_file_obj.close()\n\n err_file_obj.seek(0)\n stderr = err_file_obj.read().decode()\n err_file_obj.close()\n\n self.assertEqual(stdout, \"stdout\")\n self.assertEqual(p.stdout, b\"stdout\")\n self.assertEqual(stderr, \"stderr\")\n self.assertEqual(p.stderr, b\"stderr\")\n\n def test_tty_tee(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(\"stdout\")\n\"\"\"\n )\n read, write = pty.openpty()\n out = python(\"-u\", py.name, _out=write).stdout\n tee = os.read(read, 6)\n\n self.assertEqual(out, b\"\")\n self.assertEqual(tee, b\"stdout\")\n os.close(write)\n os.close(read)\n\n read, write = pty.openpty()\n out = python(\"-u\", py.name, _out=write, _tee=True).stdout\n tee = os.read(read, 6)\n\n self.assertEqual(out, b\"stdout\")\n self.assertEqual(tee, b\"stdout\")\n os.close(write)\n os.close(read)\n\n def test_err_redirection_actual_file(self):\n import tempfile\n\n file_obj = tempfile.NamedTemporaryFile()\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nsys.stdout.write(\"stdout\")\nsys.stderr.write(\"stderr\")\n\"\"\"\n )\n stdout = pythons(\"-u\", py.name, _err=file_obj.name)\n file_obj.seek(0)\n stderr = file_obj.read().decode()\n file_obj.close()\n self.assertEqual(stdout, \"stdout\")\n self.assertEqual(stderr, \"stderr\")\n\n def test_subcommand_and_bake(self):\n import getpass\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport subprocess\n\nprint(\"subcommand\")\nsubprocess.Popen(sys.argv[1:], shell=False).wait()\n\"\"\"\n )\n\n cmd1 = python.bake(py.name)\n out = cmd1.whoami()\n self.assertIn(\"subcommand\", out)\n self.assertIn(getpass.getuser(), out)\n\n def test_multiple_bakes(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(str(sys.argv[1:]))\n\"\"\"\n )\n\n out = python.bake(py.name).bake(\"bake1\").bake(\"bake2\")()\n self.assertEqual(\"['bake1', 'bake2']\", str(out))\n\n def test_arg_preprocessor(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(str(sys.argv[1:]))\n\"\"\"\n )\n\n def arg_preprocess(args, kwargs):\n args.insert(0, \"preprocessed\")\n kwargs[\"a-kwarg\"] = 123\n return args, kwargs\n\n cmd = pythons.bake(py.name, _arg_preprocess=arg_preprocess)\n out = cmd(\"arg\")\n self.assertEqual(\"['preprocessed', 'arg', '--a-kwarg=123']\", out)\n\n def test_bake_args_come_first(self):\n from sh import ls\n\n ls = ls.bake(h=True)\n\n ran = ls(\"-la\", _return_cmd=True).ran\n ft = ran.index(\"-h\")\n self.assertIn(\"-la\", ran[ft:])\n\n def test_output_equivalence(self):\n from sh import whoami\n\n iam1 = whoami()\n iam2 = whoami()\n\n self.assertEqual(iam1, iam2)\n\n # https://github.com/amoffat/sh/pull/252\n def test_stdout_pipe(self):\n py = create_tmp_test(\n r\"\"\"\nimport sys\n\nsys.stdout.write(\"foobar\\n\")\n\"\"\"\n )\n\n read_fd, write_fd = os.pipe()\n python(py.name, _out=write_fd, u=True)\n\n def alarm(sig, action):\n self.fail(\"Timeout while reading from pipe\")\n\n import signal\n\n signal.signal(signal.SIGALRM, alarm)\n signal.alarm(3)\n\n data = os.read(read_fd, 100)\n self.assertEqual(b\"foobar\\n\", data)\n signal.alarm(0)\n signal.signal(signal.SIGALRM, signal.SIG_DFL)\n\n def test_stdout_callback(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nfor i in range(5): print(i)\n\"\"\"\n )\n stdout = []\n\n def agg(line):\n stdout.append(line)\n\n p = python(\"-u\", py.name, _out=agg)\n p.wait()\n\n self.assertEqual(len(stdout), 5)\n\n def test_stdout_callback_no_wait(self):\n import time\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\n\nfor i in range(5):\n print(i)\n time.sleep(.5)\n\"\"\"\n )\n\n stdout = []\n\n def agg(line):\n stdout.append(line)\n\n python(\"-u\", py.name, _out=agg, _bg=True)\n\n # we give a little pause to make sure that the NamedTemporaryFile\n # exists when the python process actually starts\n time.sleep(0.5)\n\n self.assertNotEqual(len(stdout), 5)\n\n def test_stdout_callback_line_buffered(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nfor i in range(5): print(\"herpderp\")\n\"\"\"\n )\n\n stdout = []\n\n def agg(line):\n stdout.append(line)\n\n p = python(\"-u\", py.name, _out=agg, _out_bufsize=1)\n p.wait()\n\n self.assertEqual(len(stdout), 5)\n\n def test_stdout_callback_line_unbuffered(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nfor i in range(5): print(\"herpderp\")\n\"\"\"\n )\n\n stdout = []\n\n def agg(char):\n stdout.append(char)\n\n p = python(\"-u\", py.name, _out=agg, _out_bufsize=0)\n p.wait()\n\n # + 5 newlines\n self.assertEqual(len(stdout), len(\"herpderp\") * 5 + 5)\n\n def test_stdout_callback_buffered(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nfor i in range(5): sys.stdout.write(\"herpderp\")\n\"\"\"\n )\n\n stdout = []\n\n def agg(chunk):\n stdout.append(chunk)\n\n p = python(\"-u\", py.name, _out=agg, _out_bufsize=4)\n p.wait()\n\n self.assertEqual(len(stdout), len(\"herp\") / 2 * 5)\n\n def test_stdout_callback_with_input(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nfor i in range(5): print(str(i))\nderp = input(\"herp? \")\nprint(derp)\n\"\"\"\n )\n\n def agg(line, stdin):\n if line.strip() == \"4\":\n stdin.put(\"derp\\n\")\n\n p = python(\"-u\", py.name, _out=agg, _tee=True)\n p.wait()\n\n self.assertIn(\"derp\", p)\n\n def test_stdout_callback_exit(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nfor i in range(5): print(i)\n\"\"\"\n )\n\n stdout = []\n\n def agg(line):\n line = line.strip()\n stdout.append(line)\n if line == \"2\":\n return True\n\n p = python(\"-u\", py.name, _out=agg, _tee=True)\n p.wait()\n\n self.assertIn(\"4\", p)\n self.assertNotIn(\"4\", stdout)\n\n def test_stdout_callback_terminate(self):\n import signal\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\n\nfor i in range(5):\n print(i)\n time.sleep(.5)\n\"\"\"\n )\n\n stdout = []\n\n def agg(line, stdin, process):\n line = line.strip()\n stdout.append(line)\n if line == \"3\":\n process.terminate()\n return True\n\n import sh\n\n caught_signal = False\n try:\n p = python(\"-u\", py.name, _out=agg, _bg=True)\n p.wait()\n except sh.SignalException_SIGTERM:\n caught_signal = True\n\n self.assertTrue(caught_signal)\n self.assertEqual(p.process.exit_code, -signal.SIGTERM)\n self.assertNotIn(\"4\", p)\n self.assertNotIn(\"4\", stdout)\n\n def test_stdout_callback_kill(self):\n import signal\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\n\nfor i in range(5):\n print(i)\n time.sleep(.5)\n\"\"\"\n )\n\n stdout = []\n\n def agg(line, stdin, process):\n line = line.strip()\n stdout.append(line)\n if line == \"3\":\n process.kill()\n return True\n\n import sh\n\n caught_signal = False\n try:\n p = python(\"-u\", py.name, _out=agg, _bg=True)\n p.wait()\n except sh.SignalException_SIGKILL:\n caught_signal = True\n\n self.assertTrue(caught_signal)\n self.assertEqual(p.process.exit_code, -signal.SIGKILL)\n self.assertNotIn(\"4\", p)\n self.assertNotIn(\"4\", stdout)\n\n def test_general_signal(self):\n from signal import SIGINT\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\nimport signal\n\ni = 0\ndef sig_handler(sig, frame):\n global i\n i = 42\n\nsignal.signal(signal.SIGINT, sig_handler)\n\nfor _ in range(6):\n print(i)\n i += 1\n sys.stdout.flush()\n time.sleep(2)\n\"\"\"\n )\n\n stdout = []\n\n def agg(line, stdin, process):\n line = line.strip()\n stdout.append(line)\n if line == \"3\":\n process.signal(SIGINT)\n return True\n\n p = python(py.name, _out=agg, _tee=True)\n p.wait()\n\n self.assertEqual(p.process.exit_code, 0)\n self.assertEqual(str(p), \"0\\n1\\n2\\n3\\n42\\n43\\n\")\n\n def test_iter_generator(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\n\nfor i in range(42):\n print(i)\n sys.stdout.flush()\n\"\"\"\n )\n\n out = []\n for line in python(py.name, _iter=True):\n out.append(int(line.strip()))\n self.assertEqual(len(out), 42)\n self.assertEqual(sum(out), 861)\n\n def test_async(self):\n py = create_tmp_test(\n \"\"\"\nimport os\nimport time\ntime.sleep(0.5)\nprint(\"hello\")\n\"\"\"\n )\n\n alternating = []\n\n async def producer(q):\n alternating.append(1)\n msg = await python(py.name, _async=True)\n alternating.append(1)\n await q.put(msg.strip())\n\n async def consumer(q):\n await asyncio.sleep(0.1)\n alternating.append(2)\n msg = await q.get()\n self.assertEqual(msg, \"hello\")\n alternating.append(2)\n\n async def main():\n q = AQueue()\n await asyncio.gather(producer(q), consumer(q))\n\n asyncio.run(main())\n self.assertListEqual(alternating, [1, 2, 1, 2])\n\n def test_async_exc(self):\n py = create_tmp_test(\"\"\"exit(34)\"\"\")\n\n async def producer():\n await python(py.name, _async=True, _return_cmd=False)\n\n self.assertRaises(sh.ErrorReturnCode_34, asyncio.run, producer())\n\n def test_async_iter(self):\n py = create_tmp_test(\n \"\"\"\nfor i in range(5):\n print(i)\n\"\"\"\n )\n\n # this list will prove that our coroutines are yielding to eachother as each\n # line is produced\n alternating = []\n\n async def producer(q):\n async for line in python(py.name, _iter=True):\n alternating.append(1)\n await q.put(int(line.strip()))\n\n await q.put(None)\n\n async def consumer(q):\n while True:\n line = await q.get()\n if line is None:\n return\n alternating.append(2)\n\n async def main():\n q = AQueue()\n await asyncio.gather(producer(q), consumer(q))\n\n asyncio.run(main())\n self.assertListEqual(alternating, [1, 2, 1, 2, 1, 2, 1, 2, 1, 2])\n\n def test_async_iter_exc(self):\n py = create_tmp_test(\n \"\"\"\nfor i in range(5):\n print(i)\nexit(34)\n\"\"\"\n )\n\n lines = []\n\n async def producer():\n async for line in python(py.name, _async=True):\n lines.append(int(line.strip()))\n\n self.assertRaises(sh.ErrorReturnCode_34, asyncio.run, producer())\n\n def test_async_return_cmd(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.exit(0)\n\"\"\"\n )\n\n async def main():\n result = await python(py.name, _async=True, _return_cmd=True)\n self.assertIsInstance(result, sh.RunningCommand)\n result_str = await python(py.name, _async=True, _return_cmd=False)\n self.assertIsInstance(result_str, str)\n\n asyncio.run(main())\n\n def test_async_return_cmd_exc(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.exit(1)\n\"\"\"\n )\n\n async def main():\n await python(py.name, _async=True, _return_cmd=True)\n\n self.assertRaises(sh.ErrorReturnCode_1, asyncio.run, main())\n\n def test_handle_both_out_and_err(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\n\nfor i in range(42):\n sys.stdout.write(str(i) + \"\\\\n\")\n sys.stdout.flush()\n if i % 2 == 0:\n sys.stderr.write(str(i) + \"\\\\n\")\n sys.stderr.flush()\n\"\"\"\n )\n\n out = []\n\n def handle_out(line):\n out.append(int(line.strip()))\n\n err = []\n\n def handle_err(line):\n err.append(int(line.strip()))\n\n p = python(py.name, _err=handle_err, _out=handle_out, _bg=True)\n p.wait()\n\n self.assertEqual(sum(out), 861)\n self.assertEqual(sum(err), 420)\n\n def test_iter_unicode(self):\n # issue https://github.com/amoffat/sh/issues/224\n test_string = \"\\xe4\\xbd\\x95\\xe4\\xbd\\x95\\n\" * 150 # len > buffer_s\n txt = create_tmp_test(test_string)\n for line in sh.cat(txt.name, _iter=True):\n break\n self.assertLess(len(line), 1024)\n\n def test_nonblocking_iter(self):\n from errno import EWOULDBLOCK\n\n py = create_tmp_test(\n \"\"\"\nimport time\nimport sys\ntime.sleep(1)\nsys.stdout.write(\"stdout\")\n\"\"\"\n )\n count = 0\n value = None\n for line in python(py.name, _iter_noblock=True):\n if line == EWOULDBLOCK:\n count += 1\n else:\n value = line\n self.assertGreater(count, 0)\n self.assertEqual(value, \"stdout\")\n\n py = create_tmp_test(\n \"\"\"\nimport time\nimport sys\ntime.sleep(1)\nsys.stderr.write(\"stderr\")\n\"\"\"\n )\n\n count = 0\n value = None\n for line in python(py.name, _iter_noblock=\"err\"):\n if line == EWOULDBLOCK:\n count += 1\n else:\n value = line\n self.assertGreater(count, 0)\n self.assertEqual(value, \"stderr\")\n\n def test_for_generator_to_err(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nfor i in range(42):\n sys.stderr.write(str(i)+\"\\\\n\")\n\"\"\"\n )\n\n out = []\n for line in python(\"-u\", py.name, _iter=\"err\"):\n out.append(line)\n self.assertEqual(len(out), 42)\n\n # verify that nothing is going to stdout\n out = []\n for line in python(\"-u\", py.name, _iter=\"out\"):\n out.append(line)\n self.assertEqual(len(out), 0)\n\n def test_sigpipe(self):\n py1 = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\nimport signal\n\n# by default, python disables SIGPIPE, in favor of using IOError exceptions, so\n# let's put that back to the system default where we terminate with a signal\n# exit code\nsignal.signal(signal.SIGPIPE, signal.SIG_DFL)\n\nfor letter in \"andrew\":\n time.sleep(0.6)\n print(letter)\n \"\"\"\n )\n\n py2 = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\n\nwhile True:\n line = sys.stdin.readline()\n if not line:\n break\n print(line.strip().upper())\n exit(0)\n \"\"\"\n )\n\n p1 = python(\"-u\", py1.name, _piped=\"out\")\n p2 = python(\n \"-u\",\n py2.name,\n _in=p1,\n )\n\n # SIGPIPE should happen, but it shouldn't be an error, since _piped is\n # truthful\n self.assertEqual(-p1.exit_code, signal.SIGPIPE)\n self.assertEqual(p2.exit_code, 0)\n\n def test_piped_generator(self):\n import time\n\n py1 = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\n\nfor letter in \"andrew\":\n time.sleep(0.6)\n print(letter)\n \"\"\"\n )\n\n py2 = create_tmp_test(\n \"\"\"\nimport sys\nimport os\nimport time\n\nwhile True:\n line = sys.stdin.readline()\n if not line:\n break\n print(line.strip().upper())\n \"\"\"\n )\n\n times = []\n last_received = None\n\n letters = \"\"\n for line in python(\n \"-u\", py2.name, _iter=True, _in=python(\"-u\", py1.name, _piped=\"out\")\n ):\n letters += line.strip()\n\n now = time.time()\n if last_received:\n times.append(now - last_received)\n last_received = now\n\n self.assertEqual(\"ANDREW\", letters)\n self.assertTrue(all([t > 0.3 for t in times]))\n\n def test_no_out_iter_err(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stderr.write(\"1\\\\n\")\nsys.stderr.write(\"2\\\\n\")\nsys.stderr.write(\"3\\\\n\")\nsys.stderr.flush()\n\"\"\"\n )\n nums = [int(num.strip()) for num in python(py.name, _iter=\"err\", _no_out=True)]\n assert nums == [1, 2, 3]\n\n def test_generator_and_callback(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nfor i in range(42):\n sys.stderr.write(str(i * 2)+\"\\\\n\")\n print(i)\n\"\"\"\n )\n\n stderr = []\n\n def agg(line):\n stderr.append(int(line.strip()))\n\n out = []\n for line in python(\"-u\", py.name, _iter=True, _err=agg):\n out.append(line)\n\n self.assertEqual(len(out), 42)\n self.assertEqual(sum(stderr), 1722)\n\n def test_cast_bg(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport time\ntime.sleep(0.5)\nsys.stdout.write(sys.argv[1])\n\"\"\"\n )\n self.assertEqual(int(python(py.name, \"123\", _bg=True)), 123)\n self.assertEqual(float(python(py.name, \"789\", _bg=True)), 789.0)\n\n def test_cmd_eq(self):\n py = create_tmp_test(\"\")\n\n cmd1 = python.bake(py.name, \"-u\")\n cmd2 = python.bake(py.name, \"-u\")\n cmd3 = python.bake(py.name)\n\n self.assertEqual(cmd1, cmd2)\n self.assertNotEqual(cmd1, cmd3)\n\n def test_fg(self):\n py = create_tmp_test(\"exit(0)\")\n # notice we're using `system_python`, and not `python`. this is because\n # `python` has an env baked into it, and we want `_env` to be None for\n # coverage\n system_python(py.name, _fg=True)\n\n def test_fg_false(self):\n \"\"\"https://github.com/amoffat/sh/issues/520\"\"\"\n py = create_tmp_test(\"print('hello')\")\n buf = StringIO()\n python(py.name, _fg=False, _out=buf)\n self.assertEqual(buf.getvalue(), \"hello\\n\")\n\n def test_fg_true(self):\n \"\"\"https://github.com/amoffat/sh/issues/520\"\"\"\n py = create_tmp_test(\"print('hello')\")\n buf = StringIO()\n self.assertRaises(TypeError, python, py.name, _fg=True, _out=buf)\n\n def test_fg_env(self):\n py = create_tmp_test(\n \"\"\"\nimport os\ncode = int(os.environ.get(\"EXIT\", \"0\"))\nexit(code)\n\"\"\"\n )\n\n env = os.environ.copy()\n env[\"EXIT\"] = \"3\"\n self.assertRaises(sh.ErrorReturnCode_3, python, py.name, _fg=True, _env=env)\n\n def test_fg_alternative(self):\n py = create_tmp_test(\"exit(0)\")\n python(py.name, _in=sys.stdin, _out=sys.stdout, _err=sys.stderr)\n\n def test_fg_exc(self):\n py = create_tmp_test(\"exit(1)\")\n self.assertRaises(sh.ErrorReturnCode_1, python, py.name, _fg=True)\n\n def test_out_filename(self):\n outfile = tempfile.NamedTemporaryFile()\n py = create_tmp_test(\"print('output')\")\n python(py.name, _out=outfile.name)\n outfile.seek(0)\n self.assertEqual(b\"output\\n\", outfile.read())\n\n def test_out_pathlike(self):\n from pathlib import Path\n\n outfile = tempfile.NamedTemporaryFile()\n py = create_tmp_test(\"print('output')\")\n python(py.name, _out=Path(outfile.name))\n outfile.seek(0)\n self.assertEqual(b\"output\\n\", outfile.read())\n\n def test_bg_exit_code(self):\n py = create_tmp_test(\n \"\"\"\nimport time\ntime.sleep(1)\nexit(49)\n\"\"\"\n )\n p = python(py.name, _ok_code=49, _bg=True)\n self.assertEqual(49, p.exit_code)\n\n def test_cwd(self):\n from os.path import realpath\n\n from sh import pwd\n\n self.assertEqual(str(pwd(_cwd=\"/tmp\")), realpath(\"/tmp\") + \"\\n\")\n self.assertEqual(str(pwd(_cwd=\"/etc\")), realpath(\"/etc\") + \"\\n\")\n\n def test_cwd_fg(self):\n td = realpath(tempfile.mkdtemp())\n py = create_tmp_test(\n f\"\"\"\nimport sh\nimport os\nfrom os.path import realpath\norig = realpath(os.getcwd())\nprint(orig)\nsh.pwd(_cwd=\"{td}\", _fg=True)\nprint(realpath(os.getcwd()))\n\"\"\"\n )\n\n orig, newdir, restored = python(py.name).strip().split(\"\\n\")\n newdir = realpath(newdir)\n self.assertEqual(newdir, td)\n self.assertEqual(orig, restored)\n self.assertNotEqual(orig, newdir)\n os.rmdir(td)\n\n def test_huge_piped_data(self):\n from sh import tr\n\n stdin = tempfile.NamedTemporaryFile()\n\n data = \"herpderp\" * 4000 + \"\\n\"\n stdin.write(data.encode())\n stdin.flush()\n stdin.seek(0)\n\n out = tr(\"[:upper:]\", \"[:lower:]\", _in=tr(\"[:lower:]\", \"[:upper:]\", _in=data))\n self.assertTrue(out == data)\n\n def test_tty_input(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nif os.isatty(sys.stdin.fileno()):\n sys.stdout.write(\"password?\\\\n\")\n sys.stdout.flush()\n pw = sys.stdin.readline().strip()\n sys.stdout.write(\"%s\\\\n\" % (\"*\" * len(pw)))\n sys.stdout.flush()\nelse:\n sys.stdout.write(\"no tty attached!\\\\n\")\n sys.stdout.flush()\n\"\"\"\n )\n\n test_pw = \"test123\"\n expected_stars = \"*\" * len(test_pw)\n d = {}\n\n def password_enterer(line, stdin):\n line = line.strip()\n if not line:\n return\n\n if line == \"password?\":\n stdin.put(test_pw + \"\\n\")\n\n elif line.startswith(\"*\"):\n d[\"stars\"] = line\n return True\n\n pw_stars = python(py.name, _tty_in=True, _out=password_enterer)\n pw_stars.wait()\n self.assertEqual(d[\"stars\"], expected_stars)\n\n response = python(py.name)\n self.assertEqual(str(response), \"no tty attached!\\n\")\n\n def test_tty_output(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nif os.isatty(sys.stdout.fileno()):\n sys.stdout.write(\"tty attached\")\n sys.stdout.flush()\nelse:\n sys.stdout.write(\"no tty attached\")\n sys.stdout.flush()\n\"\"\"\n )\n\n out = pythons(py.name, _tty_out=True)\n self.assertEqual(out, \"tty attached\")\n\n out = pythons(py.name, _tty_out=False)\n self.assertEqual(out, \"no tty attached\")\n\n def test_stringio_output(self):\n import sh\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(sys.argv[1])\n\"\"\"\n )\n\n out = StringIO()\n sh.python(py.name, \"testing 123\", _out=out)\n self.assertEqual(out.getvalue(), \"testing 123\")\n\n out = BytesIO()\n sh.python(py.name, \"testing 123\", _out=out)\n self.assertEqual(out.getvalue().decode(), \"testing 123\")\n\n def test_stringio_input(self):\n from sh import cat\n\n input = StringIO()\n input.write(\"herpderp\")\n input.seek(0)\n\n out = cat(_in=input)\n self.assertEqual(out, \"herpderp\")\n\n def test_internal_bufsize(self):\n from sh import cat\n\n output = cat(_in=\"a\" * 1000, _internal_bufsize=100, _out_bufsize=0)\n self.assertEqual(len(output), 100)\n\n output = cat(_in=\"a\" * 1000, _internal_bufsize=50, _out_bufsize=2)\n self.assertEqual(len(output), 100)\n\n def test_change_stdout_buffering(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\n# this proves that we won't get the output into our callback until we send\n# a newline\nsys.stdout.write(\"switch \")\nsys.stdout.flush()\nsys.stdout.write(\"buffering\\\\n\")\nsys.stdout.flush()\n\nsys.stdin.read(1)\nsys.stdout.write(\"unbuffered\")\nsys.stdout.flush()\n\n# this is to keep the output from being flushed by the process ending, which\n# would ruin our test. we want to make sure we get the string \"unbuffered\"\n# before the process ends, without writing a newline\nsys.stdin.read(1)\n\"\"\"\n )\n\n d = {\n \"newline_buffer_success\": False,\n \"unbuffered_success\": False,\n }\n\n def interact(line, stdin, process):\n line = line.strip()\n if not line:\n return\n\n if line == \"switch buffering\":\n d[\"newline_buffer_success\"] = True\n process.change_out_bufsize(0)\n stdin.put(\"a\")\n\n elif line == \"unbuffered\":\n stdin.put(\"b\")\n d[\"unbuffered_success\"] = True\n return True\n\n # start with line buffered stdout\n pw_stars = python(\"-u\", py.name, _out=interact, _out_bufsize=1)\n pw_stars.wait()\n\n self.assertTrue(d[\"newline_buffer_success\"])\n self.assertTrue(d[\"unbuffered_success\"])\n\n def test_callable_interact(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(\"line1\")\n\"\"\"\n )\n\n class Callable:\n def __init__(self):\n self.line = None\n\n def __call__(self, line):\n self.line = line\n\n cb = Callable()\n python(py.name, _out=cb)\n self.assertEqual(cb.line, \"line1\")\n\n def test_encoding(self):\n return self.skipTest(\n \"what's the best way to test a different '_encoding' special keyword\"\n \"argument?\"\n )\n\n def test_timeout(self):\n from time import time\n\n import sh\n\n sleep_for = 3\n timeout = 1\n started = time()\n try:\n sh.sleep(sleep_for, _timeout=timeout).wait()\n except sh.TimeoutException as e:\n assert \"sleep 3\" in e.full_cmd\n else:\n self.fail(\"no timeout exception\")\n elapsed = time() - started\n self.assertLess(abs(elapsed - timeout), 0.5)\n\n def test_timeout_overstep(self):\n started = time.time()\n sh.sleep(1, _timeout=5)\n elapsed = time.time() - started\n self.assertLess(abs(elapsed - 1), 0.5)\n\n def test_timeout_wait(self):\n p = sh.sleep(3, _bg=True)\n self.assertRaises(sh.TimeoutException, p.wait, timeout=1)\n\n def test_timeout_wait_overstep(self):\n p = sh.sleep(1, _bg=True)\n p.wait(timeout=5)\n\n def test_timeout_wait_negative(self):\n p = sh.sleep(3, _bg=True)\n self.assertRaises(RuntimeError, p.wait, timeout=-3)\n\n def test_binary_pipe(self):\n binary = b\"\\xec;\\xedr\\xdbF\"\n\n py1 = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nsys.stdout = os.fdopen(sys.stdout.fileno(), \"wb\", 0)\nsys.stdout.write(b'\\\\xec;\\\\xedr\\\\xdbF')\n\"\"\"\n )\n\n py2 = create_tmp_test(\n \"\"\"\nimport sys\nimport os\n\nsys.stdin = os.fdopen(sys.stdin.fileno(), \"rb\", 0)\nsys.stdout = os.fdopen(sys.stdout.fileno(), \"wb\", 0)\nsys.stdout.write(sys.stdin.read())\n\"\"\"\n )\n out = python(py2.name, _in=python(py1.name))\n self.assertEqual(out.stdout, binary)\n\n # designed to trigger the \"... (%d more, please see e.stdout)\" output\n # of the ErrorReturnCode class\n def test_failure_with_large_output(self):\n from sh import ErrorReturnCode_1\n\n py = create_tmp_test(\n \"\"\"\nprint(\"andrewmoffat\" * 1000)\nexit(1)\n\"\"\"\n )\n self.assertRaises(ErrorReturnCode_1, python, py.name)\n\n # designed to check if the ErrorReturnCode constructor does not raise\n # an UnicodeDecodeError\n def test_non_ascii_error(self):\n from sh import ErrorReturnCode, ls\n\n test = \"/á\"\n self.assertRaises(ErrorReturnCode, ls, test, _encoding=\"utf8\")\n\n def test_no_out(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(\"stdout\")\nsys.stderr.write(\"stderr\")\n\"\"\"\n )\n p = python(py.name, _no_out=True)\n self.assertEqual(p.stdout, b\"\")\n self.assertEqual(p.stderr, b\"stderr\")\n self.assertTrue(p.process._pipe_queue.empty())\n\n def callback(line):\n pass\n\n p = python(py.name, _out=callback)\n self.assertEqual(p.stdout, b\"\")\n self.assertEqual(p.stderr, b\"stderr\")\n self.assertTrue(p.process._pipe_queue.empty())\n\n p = python(py.name)\n self.assertEqual(p.stdout, b\"stdout\")\n self.assertEqual(p.stderr, b\"stderr\")\n self.assertFalse(p.process._pipe_queue.empty())\n\n def test_tty_stdin(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(sys.stdin.read())\nsys.stdout.flush()\n\"\"\"\n )\n out = pythons(py.name, _in=\"test\\n\", _tty_in=True)\n self.assertEqual(\"test\\n\", out)\n\n def test_no_err(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(\"stdout\")\nsys.stderr.write(\"stderr\")\n\"\"\"\n )\n p = python(py.name, _no_err=True)\n self.assertEqual(p.stderr, b\"\")\n self.assertEqual(p.stdout, b\"stdout\")\n self.assertFalse(p.process._pipe_queue.empty())\n\n def callback(line):\n pass\n\n p = python(py.name, _err=callback)\n self.assertEqual(p.stderr, b\"\")\n self.assertEqual(p.stdout, b\"stdout\")\n self.assertFalse(p.process._pipe_queue.empty())\n\n p = python(py.name)\n self.assertEqual(p.stderr, b\"stderr\")\n self.assertEqual(p.stdout, b\"stdout\")\n self.assertFalse(p.process._pipe_queue.empty())\n\n def test_no_pipe(self):\n from sh import ls\n\n # calling a command regular should fill up the pipe_queue\n p = ls(_return_cmd=True)\n self.assertFalse(p.process._pipe_queue.empty())\n\n # calling a command with a callback should not\n def callback(line):\n pass\n\n p = ls(_out=callback, _return_cmd=True)\n self.assertTrue(p.process._pipe_queue.empty())\n\n # calling a command regular with no_pipe also should not\n p = ls(_no_pipe=True, _return_cmd=True)\n self.assertTrue(p.process._pipe_queue.empty())\n\n def test_decode_error_handling(self):\n from functools import partial\n\n py = create_tmp_test(\n \"\"\"\n# -*- coding: utf8 -*-\nimport sys\nimport os\nsys.stdout = os.fdopen(sys.stdout.fileno(), 'wb')\nsys.stdout.write(bytes(\"te漢字st\", \"utf8\") + \"äåéë\".encode(\"latin_1\"))\n\"\"\"\n )\n fn = partial(pythons, py.name, _encoding=\"ascii\")\n self.assertRaises(UnicodeDecodeError, fn)\n\n p = pythons(py.name, _encoding=\"ascii\", _decode_errors=\"ignore\")\n self.assertEqual(p, \"test\")\n\n p = pythons(\n py.name,\n _encoding=\"ascii\",\n _decode_errors=\"ignore\",\n _out=sys.stdout,\n _tee=True,\n )\n self.assertEqual(p, \"test\")\n\n def test_signal_exception(self):\n from sh import SignalException_15\n\n def throw_terminate_signal():\n py = create_tmp_test(\n \"\"\"\nimport time\nwhile True: time.sleep(1)\n\"\"\"\n )\n to_kill = python(py.name, _bg=True)\n to_kill.terminate()\n to_kill.wait()\n\n self.assertRaises(SignalException_15, throw_terminate_signal)\n\n def test_signal_group(self):\n child = create_tmp_test(\n \"\"\"\nimport time\ntime.sleep(3)\n\"\"\"\n )\n\n parent = create_tmp_test(\n \"\"\"\nimport sys\nimport sh\npython = sh.Command(sys.executable)\np = python(\"{child_file}\", _bg=True, _new_session=False)\nprint(p.pid)\nprint(p.process.pgid)\np.wait()\n\"\"\",\n child_file=child.name,\n )\n\n def launch():\n p = python(parent.name, _bg=True, _iter=True, _new_group=True)\n child_pid = int(next(p).strip())\n child_pgid = int(next(p).strip())\n parent_pid = p.pid\n parent_pgid = p.process.pgid\n\n return p, child_pid, child_pgid, parent_pid, parent_pgid\n\n def assert_alive(pid):\n os.kill(pid, 0)\n\n def assert_dead(pid):\n self.assert_oserror(errno.ESRCH, os.kill, pid, 0)\n\n # first let's prove that calling regular SIGKILL on the parent does\n # nothing to the child, since the child was launched in the same process\n # group (_new_session=False) and the parent is not a controlling process\n p, child_pid, child_pgid, parent_pid, parent_pgid = launch()\n\n assert_alive(parent_pid)\n assert_alive(child_pid)\n\n p.kill()\n time.sleep(0.1)\n assert_dead(parent_pid)\n assert_alive(child_pid)\n\n self.assertRaises(sh.SignalException_SIGKILL, p.wait)\n assert_dead(child_pid)\n\n # now let's prove that killing the process group kills both the parent\n # and the child\n p, child_pid, child_pgid, parent_pid, parent_pgid = launch()\n\n assert_alive(parent_pid)\n assert_alive(child_pid)\n\n p.kill_group()\n time.sleep(0.1)\n assert_dead(parent_pid)\n assert_dead(child_pid)\n\n def test_pushd(self):\n \"\"\"test basic pushd functionality\"\"\"\n child = realpath(tempfile.mkdtemp())\n\n old_wd1 = sh.pwd().strip()\n old_wd2 = os.getcwd()\n\n self.assertEqual(old_wd1, old_wd2)\n self.assertNotEqual(old_wd1, child)\n\n with sh.pushd(child):\n new_wd1 = sh.pwd().strip()\n new_wd2 = os.getcwd()\n\n old_wd3 = sh.pwd().strip()\n old_wd4 = os.getcwd()\n self.assertEqual(old_wd3, old_wd4)\n self.assertEqual(old_wd1, old_wd3)\n\n self.assertEqual(new_wd1, child)\n self.assertEqual(new_wd2, child)\n\n def test_pushd_cd(self):\n \"\"\"test that pushd works like pushd/popd\"\"\"\n child = realpath(tempfile.mkdtemp())\n try:\n old_wd = os.getcwd()\n with sh.pushd(tempdir):\n self.assertEqual(str(tempdir), os.getcwd())\n self.assertEqual(old_wd, os.getcwd())\n finally:\n os.rmdir(child)\n\n def test_non_existant_cwd(self):\n from sh import ls\n\n # sanity check\n non_exist_dir = join(tempdir, \"aowjgoahewro\")\n self.assertFalse(exists(non_exist_dir))\n self.assertRaises(sh.ForkException, ls, _cwd=non_exist_dir)\n\n # https://github.com/amoffat/sh/issues/176\n def test_baked_command_can_be_printed(self):\n from sh import ls\n\n ll = ls.bake(\"-l\")\n self.assertTrue(str(ll).endswith(\"/ls -l\"))\n\n def test_baked_command_can_be_printed_with_whitespace_args(self):\n from sh import ls\n\n ls_himym = ls.bake(\"How I Met Your Mother\")\n self.assertTrue(str(ls_himym).endswith(\"/ls 'How I Met Your Mother'\"))\n ls_himym = ls.bake(\"How I 'Met' Your Mother\")\n self.assertTrue(\n str(ls_himym).endswith(\"\"\"/ls 'How I '\"'\"'Met'\"'\"' Your Mother'\"\"\")\n )\n ls_himym = ls.bake('How I \"Met\" Your Mother')\n self.assertTrue(str(ls_himym).endswith(\"\"\"/ls 'How I \"Met\" Your Mother'\"\"\"))\n\n def test_baked_command_can_be_printed_with_whitespace_in_options(self):\n from sh import ls\n\n cmd = ls.bake(o=\"one two\")\n self.assertTrue(str(cmd).endswith(\"\"\"/ls -o 'one two'\"\"\"), str(cmd))\n cmd = ls.bake(opt=\"one two\")\n self.assertTrue(str(cmd).endswith(\"\"\"/ls '--opt=one two'\"\"\"), str(cmd))\n\n # https://github.com/amoffat/sh/issues/185\n def test_done_callback(self):\n import time\n\n class Callback:\n def __init__(self):\n self.called = False\n self.exit_code = None\n self.success = None\n\n def __call__(self, p, success, exit_code):\n self.called = True\n self.exit_code = exit_code\n self.success = success\n\n py = create_tmp_test(\n \"\"\"\nfrom time import time, sleep\nsleep(1)\nprint(time())\n\"\"\"\n )\n\n callback = Callback()\n p = python(py.name, _done=callback, _bg=True)\n\n # do a little setup to prove that a command with a _done callback is run\n # in the background\n wait_start = time.time()\n p.wait()\n wait_elapsed = time.time() - wait_start\n\n self.assertTrue(callback.called)\n self.assertLess(abs(wait_elapsed - 1.0), 1.0)\n self.assertEqual(callback.exit_code, 0)\n self.assertTrue(callback.success)\n\n # https://github.com/amoffat/sh/issues/564\n def test_done_callback_no_deadlock(self):\n import time\n\n py = create_tmp_test(\n \"\"\"\nfrom sh import sleep\n\ndef done(cmd, success, exit_code):\n print(cmd, success, exit_code)\n\nsleep('1', _done=done)\n\"\"\"\n )\n\n p = python(py.name, _bg=True, _timeout=2)\n\n # do a little setup to prove that a command with a _done callback is run\n # in the background\n wait_start = time.time()\n p.wait()\n wait_elapsed = time.time() - wait_start\n\n self.assertLess(abs(wait_elapsed - 1.0), 1.0)\n\n def test_fork_exc(self):\n from sh import ForkException\n\n py = create_tmp_test(\"\")\n\n def fail():\n raise RuntimeError(\"nooo\")\n\n self.assertRaises(ForkException, python, py.name, _preexec_fn=fail)\n\n def test_new_session_new_group(self):\n from threading import Event\n\n py = create_tmp_test(\n \"\"\"\nimport os\nimport time\npid = os.getpid()\npgid = os.getpgid(pid)\nsid = os.getsid(pid)\nstuff = [pid, pgid, sid]\n\nprint(\",\".join([str(el) for el in stuff]))\ntime.sleep(0.5)\n\"\"\"\n )\n\n event = Event()\n\n def handle(run_asserts, line, stdin, p):\n pid, pgid, sid = line.strip().split(\",\")\n pid = int(pid)\n pgid = int(pgid)\n sid = int(sid)\n test_pid = os.getpgid(os.getpid())\n\n self.assertEqual(p.pid, pid)\n self.assertEqual(p.pgid, pgid)\n self.assertEqual(pgid, p.get_pgid())\n self.assertEqual(p.sid, sid)\n self.assertEqual(sid, p.get_sid())\n\n run_asserts(pid, pgid, sid, test_pid)\n event.set()\n\n def session_true_group_false(pid, pgid, sid, test_pid):\n self.assertEqual(pid, sid)\n self.assertEqual(pid, pgid)\n\n p = python(\n py.name, _out=partial(handle, session_true_group_false), _new_session=True\n )\n p.wait()\n self.assertTrue(event.is_set())\n\n event.clear()\n\n def session_false_group_false(pid, pgid, sid, test_pid):\n self.assertEqual(test_pid, pgid)\n self.assertNotEqual(pid, sid)\n\n p = python(\n py.name, _out=partial(handle, session_false_group_false), _new_session=False\n )\n p.wait()\n self.assertTrue(event.is_set())\n\n event.clear()\n\n def session_false_group_true(pid, pgid, sid, test_pid):\n self.assertEqual(pid, pgid)\n self.assertNotEqual(pid, sid)\n\n p = python(\n py.name,\n _out=partial(handle, session_false_group_true),\n _new_session=False,\n _new_group=True,\n )\n p.wait()\n self.assertTrue(event.is_set())\n\n event.clear()\n\n def test_done_cb_exc(self):\n from sh import ErrorReturnCode\n\n class Callback:\n def __init__(self):\n self.called = False\n self.success = None\n\n def __call__(self, p, success, exit_code):\n self.success = success\n self.called = True\n\n py = create_tmp_test(\"exit(1)\")\n\n callback = Callback()\n try:\n p = python(py.name, _done=callback, _bg=True)\n p.wait()\n except ErrorReturnCode:\n self.assertTrue(callback.called)\n self.assertFalse(callback.success)\n else:\n self.fail(\"command should've thrown an exception\")\n\n def test_callable_stdin(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(sys.stdin.read())\n\"\"\"\n )\n\n def create_stdin():\n state = {\"count\": 0}\n\n def stdin():\n count = state[\"count\"]\n if count == 4:\n return None\n state[\"count\"] += 1\n return str(count)\n\n return stdin\n\n out = pythons(py.name, _in=create_stdin())\n self.assertEqual(\"0123\", out)\n\n def test_stdin_unbuffered_bufsize(self):\n from time import sleep\n\n # this tries to receive some known data and measures the time it takes\n # to receive it. since we're flushing by newline, we should only be\n # able to receive the data when a newline is fed in\n py = create_tmp_test(\n \"\"\"\nimport sys\nfrom time import time\n\nstarted = time()\ndata = sys.stdin.read(len(\"testing\"))\nwaited = time() - started\nsys.stdout.write(data + \"\\\\n\")\nsys.stdout.write(str(waited) + \"\\\\n\")\n\nstarted = time()\ndata = sys.stdin.read(len(\"done\"))\nwaited = time() - started\nsys.stdout.write(data + \"\\\\n\")\nsys.stdout.write(str(waited) + \"\\\\n\")\n\nsys.stdout.flush()\n\"\"\"\n )\n\n def create_stdin():\n yield \"test\"\n sleep(1)\n yield \"ing\"\n sleep(1)\n yield \"done\"\n\n out = python(py.name, _in=create_stdin(), _in_bufsize=0)\n word1, time1, word2, time2, _ = out.split(\"\\n\")\n time1 = float(time1)\n time2 = float(time2)\n self.assertEqual(word1, \"testing\")\n self.assertLess(abs(1 - time1), 0.5)\n self.assertEqual(word2, \"done\")\n self.assertLess(abs(1 - time2), 0.5)\n\n def test_stdin_newline_bufsize(self):\n from time import sleep\n\n # this tries to receive some known data and measures the time it takes\n # to receive it. since we're flushing by newline, we should only be\n # able to receive the data when a newline is fed in\n py = create_tmp_test(\n \"\"\"\nimport sys\nfrom time import time\n\nstarted = time()\ndata = sys.stdin.read(len(\"testing\\\\n\"))\nwaited = time() - started\nsys.stdout.write(data)\nsys.stdout.write(str(waited) + \"\\\\n\")\n\nstarted = time()\ndata = sys.stdin.read(len(\"done\\\\n\"))\nwaited = time() - started\nsys.stdout.write(data)\nsys.stdout.write(str(waited) + \"\\\\n\")\n\nsys.stdout.flush()\n\"\"\"\n )\n\n # we'll feed in text incrementally, sleeping strategically before\n # sending a newline. we then measure the amount that we slept\n # indirectly in the child process\n def create_stdin():\n yield \"test\"\n sleep(1)\n yield \"ing\\n\"\n sleep(1)\n yield \"done\\n\"\n\n out = python(py.name, _in=create_stdin(), _in_bufsize=1)\n word1, time1, word2, time2, _ = out.split(\"\\n\")\n time1 = float(time1)\n time2 = float(time2)\n self.assertEqual(word1, \"testing\")\n self.assertLess(abs(1 - time1), 0.5)\n self.assertEqual(word2, \"done\")\n self.assertLess(abs(1 - time2), 0.5)\n\n def test_custom_timeout_signal(self):\n import signal\n\n from sh import TimeoutException\n\n py = create_tmp_test(\n \"\"\"\nimport time\ntime.sleep(3)\n\"\"\"\n )\n try:\n python(py.name, _timeout=1, _timeout_signal=signal.SIGHUP)\n except TimeoutException as e:\n self.assertEqual(e.exit_code, signal.SIGHUP)\n else:\n self.fail(\"we should have handled a TimeoutException\")\n\n def test_timeout_race_condition_process_exit(self):\n import signal\n\n from sh import TimeoutException\n\n py = create_tmp_test(\n \"\"\"\nimport time\ntime.sleep(0.05)\n\"\"\"\n )\n # Run multiple times to increase likelihood of hitting the race condition\n for _ in range(50):\n try:\n python(py.name, _timeout=0.1, _timeout_signal=signal.SIGTERM)\n except TimeoutException:\n # TimeoutException is OK, but ProcessLookupError isn't\n pass\n\n def test_append_stdout(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nnum = sys.stdin.read()\nsys.stdout.write(num)\n\"\"\"\n )\n append_file = tempfile.NamedTemporaryFile(mode=\"a+b\")\n python(py.name, _in=\"1\", _out=append_file)\n python(py.name, _in=\"2\", _out=append_file)\n append_file.seek(0)\n output = append_file.read()\n self.assertEqual(b\"12\", output)\n\n def test_shadowed_subcommand(self):\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(sys.argv[1])\n\"\"\"\n )\n out = pythons.bake(py.name).bake_()\n self.assertEqual(\"bake\", out)\n\n def test_no_proc_no_attr(self):\n py = create_tmp_test(\"\")\n with python(py.name) as p:\n self.assertRaises(AttributeError, getattr, p, \"exit_code\")\n\n def test_partially_applied_callback(self):\n from functools import partial\n\n py = create_tmp_test(\n \"\"\"\nfor i in range(10):\n print(i)\n\"\"\"\n )\n\n output = []\n\n def fn(foo, line):\n output.append((foo, int(line.strip())))\n\n log_line = partial(fn, \"hello\")\n python(py.name, _out=log_line)\n self.assertEqual(output, [(\"hello\", i) for i in range(10)])\n\n output = []\n\n def fn(foo, line, stdin, proc):\n output.append((foo, int(line.strip())))\n\n log_line = partial(fn, \"hello\")\n python(py.name, _out=log_line)\n self.assertEqual(output, [(\"hello\", i) for i in range(10)])\n\n # https://github.com/amoffat/sh/issues/266\n def test_grandchild_no_sighup(self):\n import time\n\n # child process that will write to a file if it receives a SIGHUP\n child = create_tmp_test(\n \"\"\"\nimport signal\nimport sys\nimport time\n\noutput_file = sys.argv[1]\nwith open(output_file, \"w\") as f:\n def handle_sighup(signum, frame):\n f.write(\"got signal %d\" % signum)\n sys.exit(signum)\n signal.signal(signal.SIGHUP, handle_sighup)\n time.sleep(2)\n f.write(\"made it!\\\\n\")\n\"\"\"\n )\n\n # the parent that will terminate before the child writes to the output\n # file, potentially causing a SIGHUP\n parent = create_tmp_test(\n \"\"\"\nimport os\nimport time\nimport sys\n\nchild_file = sys.argv[1]\noutput_file = sys.argv[2]\n\npython_name = os.path.basename(sys.executable)\nos.spawnlp(os.P_NOWAIT, python_name, python_name, child_file, output_file)\ntime.sleep(1) # give child a chance to set up\n\"\"\"\n )\n\n output_file = tempfile.NamedTemporaryFile(delete=True)\n python(parent.name, child.name, output_file.name)\n time.sleep(3)\n\n out = output_file.readlines()[0]\n self.assertEqual(out, b\"made it!\\n\")\n\n def test_unchecked_producer_failure(self):\n from sh import ErrorReturnCode_2\n\n producer = create_tmp_test(\n \"\"\"\nimport sys\nfor i in range(10):\n print(i)\nsys.exit(2)\n\"\"\"\n )\n\n consumer = create_tmp_test(\n \"\"\"\nimport sys\nfor line in sys.stdin:\n pass\n\"\"\"\n )\n\n direct_pipe = python(producer.name, _piped=True)\n self.assertRaises(ErrorReturnCode_2, python, direct_pipe, consumer.name)\n\n def test_unchecked_pipeline_failure(self):\n # similar to test_unchecked_producer_failure, but this\n # tests a multi-stage pipeline\n\n from sh import ErrorReturnCode_2\n\n producer = create_tmp_test(\n \"\"\"\nimport sys\nfor i in range(10):\n print(i)\nsys.exit(2)\n\"\"\"\n )\n\n middleman = create_tmp_test(\n \"\"\"\nimport sys\nfor line in sys.stdin:\n print(\"> \" + line)\n\"\"\"\n )\n\n consumer = create_tmp_test(\n \"\"\"\nimport sys\nfor line in sys.stdin:\n pass\n\"\"\"\n )\n\n producer_normal_pipe = python(producer.name, _piped=True)\n middleman_normal_pipe = python(\n middleman.name, _piped=True, _in=producer_normal_pipe\n )\n self.assertRaises(\n ErrorReturnCode_2, python, middleman_normal_pipe, consumer.name\n )\n\n def test_bad_sig_raise_exception(self):\n # test all bad signal are correctly raised\n py = create_tmp_test(\n \"\"\"\nimport time\nimport sys\n\ntime.sleep(2)\nsys.exit(1)\n\"\"\"\n )\n for sig in SIGNALS_THAT_SHOULD_THROW_EXCEPTION:\n if sig == signal.SIGPIPE:\n continue\n sig_exception_name = f\"SignalException_{sig}\"\n sig_exception = getattr(sh, sig_exception_name)\n try:\n p = python_bg(py.name)\n time.sleep(0.5)\n p.signal(sig)\n p.wait()\n except sig_exception:\n pass\n else:\n self.fail(f\"{sig_exception_name} not raised\")\n\n def test_ok_code_ignores_bad_sig_exception(self):\n # Test if I have [-sig] in _ok_code, the exception won't be raised\n py = create_tmp_test(\n \"\"\"\nimport time\nimport sys\n\ntime.sleep(2)\nsys.exit(1)\n\"\"\"\n )\n for sig in SIGNALS_THAT_SHOULD_THROW_EXCEPTION:\n if sig == signal.SIGPIPE:\n continue\n sig_exception_name = f\"SignalException_{sig}\"\n sig_exception = getattr(sh, sig_exception_name)\n python_bg_no_sig_exception = python_bg.bake(_ok_code=[-sig])\n try:\n p = python_bg_no_sig_exception(py.name)\n time.sleep(0.5)\n p.signal(sig)\n p.wait()\n except sig_exception:\n self.fail(\n f\"{sig_exception_name} should not be raised setting _ok_code.\"\n )\n else:\n self.assertEqual(p.exit_code, -sig)\n\n\nclass MockTests(BaseTests):\n def test_patch_command_cls(self):\n def fn():\n cmd = sh.Command(\"afowejfow\")\n return cmd()\n\n @unittest.mock.patch(\"sh.Command\")\n def test(Command):\n Command().return_value = \"some output\"\n return fn()\n\n self.assertEqual(test(), \"some output\")\n self.assertRaises(sh.CommandNotFound, fn)\n\n def test_patch_command(self):\n def fn():\n return sh.afowejfow()\n\n @unittest.mock.patch(\"sh.afowejfow\", create=True)\n def test(cmd):\n cmd.return_value = \"some output\"\n return fn()\n\n self.assertEqual(test(), \"some output\")\n self.assertRaises(sh.CommandNotFound, fn)\n\n\nclass MiscTests(BaseTests):\n def test_pickling(self):\n import pickle\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(\"some output\")\nsys.stderr.write(\"some error\")\nexit(1)\n\"\"\"\n )\n\n try:\n python(py.name)\n except sh.ErrorReturnCode as e:\n restored = pickle.loads(pickle.dumps(e))\n self.assertEqual(restored.stdout, b\"some output\")\n self.assertEqual(restored.stderr, b\"some error\")\n self.assertEqual(restored.exit_code, 1)\n else:\n self.fail(\"Didn't get an exception\")\n\n @requires_poller(\"poll\")\n def test_fd_over_1024(self):\n py = create_tmp_test(\"\"\"print(\"hi world\")\"\"\")\n\n with ulimit(resource.RLIMIT_NOFILE, 2048):\n cutoff_fd = 1024\n pipes = []\n for i in range(cutoff_fd):\n master, slave = os.pipe()\n pipes.append((master, slave))\n if slave >= cutoff_fd:\n break\n\n python(py.name)\n for master, slave in pipes:\n os.close(master)\n os.close(slave)\n\n def test_args_deprecated(self):\n self.assertRaises(DeprecationWarning, sh.args, _env={})\n\n def test_percent_doesnt_fail_logging(self):\n \"\"\"test that a command name doesn't interfere with string formatting in\n the internal loggers\"\"\"\n py = create_tmp_test(\n \"\"\"\nprint(\"cool\")\n\"\"\"\n )\n python(py.name, \"%\")\n python(py.name, \"%%\")\n python(py.name, \"%%%\")\n\n def test_pushd_thread_safety(self):\n import threading\n import time\n\n temp1 = realpath(tempfile.mkdtemp())\n temp2 = realpath(tempfile.mkdtemp())\n try:\n results = [None, None]\n\n def fn1():\n with sh.pushd(temp1):\n time.sleep(0.2)\n results[0] = realpath(os.getcwd())\n\n def fn2():\n time.sleep(0.1)\n with sh.pushd(temp2):\n results[1] = realpath(os.getcwd())\n time.sleep(0.3)\n\n t1 = threading.Thread(name=\"t1\", target=fn1)\n t2 = threading.Thread(name=\"t2\", target=fn2)\n\n t1.start()\n t2.start()\n\n t1.join()\n t2.join()\n\n self.assertEqual(results, [temp1, temp2])\n finally:\n os.rmdir(temp1)\n os.rmdir(temp2)\n\n def test_stdin_nohang(self):\n py = create_tmp_test(\n \"\"\"\nprint(\"hi\")\n\"\"\"\n )\n read, write = os.pipe()\n stdin = os.fdopen(read, \"r\")\n python(py.name, _in=stdin)\n\n @requires_utf8\n def test_unicode_path(self):\n from sh import Command\n\n python_name = os.path.basename(sys.executable)\n py = create_tmp_test(\n f\"\"\"#!/usr/bin/env {python_name}\n# -*- coding: utf8 -*-\nprint(\"字\")\n\"\"\",\n prefix=\"字\",\n delete=False,\n )\n\n try:\n py.close()\n os.chmod(py.name, int(0o755))\n cmd = Command(py.name)\n\n # all of these should behave just fine\n str(cmd)\n repr(cmd)\n\n running = cmd(_return_cmd=True)\n str(running)\n repr(running)\n\n str(running.process)\n repr(running.process)\n\n finally:\n os.unlink(py.name)\n\n # https://github.com/amoffat/sh/issues/121\n def test_wraps(self):\n from sh import ls\n\n wraps(ls)(lambda f: True)\n\n def test_signal_exception_aliases(self):\n \"\"\"proves that signal exceptions with numbers and names are equivalent\"\"\"\n import signal\n\n import sh\n\n sig_name = f\"SignalException_{signal.SIGQUIT}\"\n sig = getattr(sh, sig_name)\n from sh import SignalException_SIGQUIT\n\n self.assertEqual(sig, SignalException_SIGQUIT)\n\n def test_change_log_message(self):\n py = create_tmp_test(\n \"\"\"\nprint(\"cool\")\n\"\"\"\n )\n\n def log_msg(cmd, call_args, pid=None):\n return \"Hi! I ran something\"\n\n buf = StringIO()\n handler = logging.StreamHandler(buf)\n logger = logging.getLogger(\"sh\")\n logger.setLevel(logging.INFO)\n\n try:\n logger.addHandler(handler)\n python(py.name, \"meow\", \"bark\", _log_msg=log_msg)\n finally:\n logger.removeHandler(handler)\n\n loglines = buf.getvalue().split(\"\\n\")\n self.assertTrue(loglines, \"Log handler captured no messages?\")\n self.assertTrue(loglines[0].startswith(\"Hi! I ran something\"))\n\n # https://github.com/amoffat/sh/issues/273\n def test_stop_iteration_doesnt_block(self):\n \"\"\"proves that calling calling next() on a stopped iterator doesn't\n hang.\"\"\"\n py = create_tmp_test(\n \"\"\"\nprint(\"cool\")\n\"\"\"\n )\n p = python(py.name, _iter=True)\n for i in range(100):\n try:\n next(p)\n except StopIteration:\n pass\n\n # https://github.com/amoffat/sh/issues/195\n def test_threaded_with_contexts(self):\n import threading\n import time\n\n py = create_tmp_test(\n \"\"\"\nimport sys\na = sys.argv\nres = (a[1], a[3])\nsys.stdout.write(repr(res))\n\"\"\"\n )\n\n p1 = python.bake(\"-u\", py.name, 1)\n p2 = python.bake(\"-u\", py.name, 2)\n results = [None, None]\n\n def f1():\n with p1:\n time.sleep(1)\n results[0] = str(system_python(\"one\"))\n\n def f2():\n with p2:\n results[1] = str(system_python(\"two\"))\n\n t1 = threading.Thread(target=f1)\n t1.start()\n\n t2 = threading.Thread(target=f2)\n t2.start()\n\n t1.join()\n t2.join()\n\n correct = [\n \"('1', 'one')\",\n \"('2', 'two')\",\n ]\n self.assertEqual(results, correct)\n\n # https://github.com/amoffat/sh/pull/292\n def test_eintr(self):\n import signal\n\n def handler(num, frame):\n pass\n\n signal.signal(signal.SIGALRM, handler)\n\n py = create_tmp_test(\n \"\"\"\nimport time\ntime.sleep(2)\n\"\"\"\n )\n p = python(py.name, _bg=True)\n signal.alarm(1)\n p.wait()\n\n\nclass StreamBuffererTests(unittest.TestCase):\n def test_unbuffered(self):\n from sh import StreamBufferer\n\n b = StreamBufferer(0)\n\n self.assertEqual(b.process(b\"test\"), [b\"test\"])\n self.assertEqual(b.process(b\"one\"), [b\"one\"])\n self.assertEqual(b.process(b\"\"), [b\"\"])\n self.assertEqual(b.flush(), b\"\")\n\n def test_newline_buffered(self):\n from sh import StreamBufferer\n\n b = StreamBufferer(1)\n\n self.assertEqual(b.process(b\"testing\\none\\ntwo\"), [b\"testing\\n\", b\"one\\n\"])\n self.assertEqual(b.process(b\"\\nthree\\nfour\"), [b\"two\\n\", b\"three\\n\"])\n self.assertEqual(b.flush(), b\"four\")\n\n def test_chunk_buffered(self):\n from sh import StreamBufferer\n\n b = StreamBufferer(10)\n\n self.assertEqual(b.process(b\"testing\\none\\ntwo\"), [b\"testing\\non\"])\n self.assertEqual(b.process(b\"\\nthree\\n\"), [b\"e\\ntwo\\nthre\"])\n self.assertEqual(b.flush(), b\"e\\n\")\n\n\n@requires_posix\nclass ExecutionContextTests(unittest.TestCase):\n def test_basic(self):\n import sh\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(sys.argv[1])\n\"\"\"\n )\n\n out = StringIO()\n sh2 = sh.bake(_out=out)\n sh2.python(py.name, \"TEST\")\n self.assertEqual(\"TEST\", out.getvalue())\n\n def test_multiline_defaults(self):\n py = create_tmp_test(\n \"\"\"\nimport os\nprint(os.environ[\"ABC\"])\n\"\"\"\n )\n\n sh2 = sh.bake(\n _env={\n \"ABC\": \"123\",\n }\n )\n output = sh2.python(py.name).strip()\n assert output == \"123\"\n\n def test_no_interfere1(self):\n import sh\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(sys.argv[1])\n\"\"\"\n )\n\n out = StringIO()\n _sh = sh.bake(_out=out) # noqa: F841\n\n _sh.python(py.name, \"TEST\")\n self.assertEqual(\"TEST\", out.getvalue())\n\n # Emptying the StringIO\n out.seek(0)\n out.truncate(0)\n\n sh.python(py.name, \"KO\")\n self.assertEqual(\"\", out.getvalue())\n\n def test_no_interfere2(self):\n import sh\n\n out = StringIO()\n from sh import echo\n\n _sh = sh.bake(_out=out) # noqa: F841\n echo(\"-n\", \"TEST\")\n self.assertEqual(\"\", out.getvalue())\n\n def test_set_in_parent_function(self):\n import sh\n\n py = create_tmp_test(\n \"\"\"\nimport sys\nsys.stdout.write(sys.argv[1])\n\"\"\"\n )\n\n out = StringIO()\n _sh = sh.bake(_out=out)\n\n def nested1():\n _sh.python(py.name, \"TEST1\")\n\n def nested2():\n import sh\n\n sh.python(py.name, \"TEST2\")\n\n nested1()\n nested2()\n self.assertEqual(\"TEST1\", out.getvalue())\n\n def test_command_with_baked_call_args(self):\n # Test that sh.Command() knows about baked call args\n import sh\n\n _sh = sh.bake(_ok_code=1)\n self.assertEqual(sh.Command._call_args[\"ok_code\"], 0)\n self.assertEqual(_sh.Command._call_args[\"ok_code\"], 1)\n\n\nif __name__ == \"__main__\":\n root = logging.getLogger()\n root.setLevel(logging.DEBUG)\n root.addHandler(logging.NullHandler())\n\n test_kwargs = {\"warnings\": \"ignore\"}\n\n # if we're running a specific test, we can let unittest framework figure out\n # that test and run it itself. it will also handle setting the return code\n # of the process if any tests error or fail\n if len(sys.argv) > 1:\n unittest.main(**test_kwargs)\n\n # otherwise, it looks like we want to run all the tests\n else:\n suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])\n test_kwargs[\"verbosity\"] = 2\n result = unittest.TextTestRunner(**test_kwargs).run(suite)\n\n if not result.wasSuccessful():\n exit(1)\n" }, { "path": "tox.ini", "content": "[tox]\nenvlist = py{38,39,310,311}-locale-{c,utf8}-poller-{poll,select},lint\nisolated_build = True\n\n[testenv]\nallowlist_externals = poetry\nsetenv =\n locale-c: LANG=C\n locale-utf8: LANG=en_US.UTF-8\n poller-select: SH_TESTS_USE_SELECT=1\n poller-poll: SH_TESTS_USE_SELECT=0\n SH_TESTS_RUNNING=1\ncommands =\n python sh_test.py {posargs}\n\n[testenv:lint]\nallowlist_externals =\n flake8\n black\n rstcheck\n mypy\ncommands =\n flake8 sh.py sh_test.py\n black --check --diff sh.py sh_test.py\n rstcheck README.rst\n mypy sh.py" } ]