Repository: amoffat/sh Branch: develop Commit: 2a90b1f87a87 Files: 50 Total size: 340.7 KB Directory structure: gitextract_9g2q4bcd/ ├── .coveragerc ├── .flake8 ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ └── main.yml ├── .gitignore ├── .python-version ├── .readthedocs.yaml ├── .vscode/ │ └── tasks.json ├── CHANGELOG.md ├── CODEOWNERS ├── LICENSE.txt ├── MIGRATION.md ├── Makefile ├── README.rst ├── dev_scripts/ │ └── changelog_extract.py ├── docs/ │ ├── Makefile │ ├── requirements.txt │ └── source/ │ ├── conf.py │ ├── examples/ │ │ └── done.rst │ ├── fulldoc.rst │ ├── index.rst │ ├── ref_to_fulldoc.rst │ ├── reference.rst │ ├── sections/ │ │ ├── architecture.rst │ │ ├── asynchronous_execution.rst │ │ ├── baking.rst │ │ ├── command_class.rst │ │ ├── contrib.rst │ │ ├── default_arguments.rst │ │ ├── envs.rst │ │ ├── exit_codes.rst │ │ ├── faq.rst │ │ ├── passing_arguments.rst │ │ ├── piping.rst │ │ ├── redirection.rst │ │ ├── special_arguments.rst │ │ ├── stdin.rst │ │ ├── subcommands.rst │ │ ├── sudo.rst │ │ └── with.rst │ ├── tutorials/ │ │ ├── interacting_with_processes.rst │ │ └── real_time_output.rst │ ├── tutorials.rst │ └── usage.rst ├── pyproject.toml ├── sh.py ├── tests/ │ ├── Dockerfile │ ├── __init__.py │ └── sh_test.py └── tox.ini ================================================ FILE CONTENTS ================================================ ================================================ FILE: .coveragerc ================================================ [run] branch = True source = sh relative_files = True [report] exclude_lines = pragma: no cover if __name__ == .__main__.: def __repr__ ================================================ FILE: .flake8 ================================================ [flake8] max-line-length = 88 extend-ignore = E203 ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: [ecederstrand, amoffat] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] ================================================ FILE: .github/workflows/main.yml ================================================ # This workflow will install Python dependencies, run tests and converage with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Run tests on: pull_request: push: branches: - master jobs: lint: name: Lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 name: Cache pip directory with: path: ~/.cache/pip key: ${{ runner.os }}-pip-3.9 - uses: actions/cache@v4 name: Cache poetry deps with: path: .venv key: ${{ runner.os }}-build-${{ hashFiles('poetry.lock') }}-3.9 - name: Set up Python uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install poetry run: | pip install poetry - name: Install dependencies run: | poetry config virtualenvs.in-project true poetry install - name: Lint run: | poetry run python -m flake8 sh.py tests/*.py poetry run black --check --diff sh.py tests/*.py poetry run rstcheck README.rst poetry run mypy sh.py test: name: Run tests runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] use-select: [0, 1] lang: [C, en_US.UTF-8] steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 name: Cache pip directory with: path: ~/.cache/pip key: ${{ runner.os }}-pip-3.9 - uses: actions/cache@v4 name: Cache poetry deps env: cache-name: poetry-deps with: path: .venv key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('poetry.lock') }}-${{ matrix.python-version }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install poetry run: | pip install poetry - name: Install dependencies run: | poetry config virtualenvs.in-project true poetry install - name: Run tests run: | SH_TESTS_RUNNING=1 SH_TESTS_USE_SELECT=${{ matrix.use-select }} LANG=${{ matrix.lang }} poetry run coverage run --data-file=coverage.data -a -m pytest - name: Store coverage uses: actions/upload-artifact@v4 with: name: coverage.${{ matrix.use-select }}.${{ matrix.lang }}.${{ matrix.python-version }} path: coverage.data report: name: Report Coverage needs: test runs-on: ubuntu-latest steps: # required because coveralls complains if we're not in a git dir - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies run: | pip install coverage coveralls - name: Download coverage artifacts uses: actions/download-artifact@v4 with: path: coverage-artifacts - name: Combine coverage run: | find coverage-artifacts -name coverage.data | xargs coverage combine -a - name: Report coverage env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | coverage report coveralls --service=github deploy: name: Deploy needs: test runs-on: ubuntu-latest if: github.ref_name == 'master' permissions: contents: write id-token: write steps: - uses: actions/checkout@v4 - name: Get current version id: get_version run: | version=$(sed -n 's/^version = "\(.*\)"/\1/p' pyproject.toml) echo "version=$version" >> "$GITHUB_OUTPUT" - name: Set up Python uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies run: pip install build - name: Build run: python -m build - name: Tag commit run: | git tag "${{steps.get_version.outputs.version}}" "${{github.ref_name}}" git push -f origin "${{steps.get_version.outputs.version}}" - name: Get changes id: changelog run: | python dev_scripts/changelog_extract.py ${{ steps.get_version.outputs.version }} \ > release_changes.md - name: Create Release id: create-release uses: softprops/action-gh-release@v2 with: tag_name: ${{ steps.get_version.outputs.version }} name: Release ${{ steps.get_version.outputs.version }} body_path: release_changes.md draft: false prerelease: false files: dist/* - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 ================================================ FILE: .gitignore ================================================ __pycache__/ *.py[co] .tox .coverage /.cache/ /.venv/ /build /dist /docs/build /TODO.md /htmlcov/ ================================================ FILE: .python-version ================================================ 3.9.16 ================================================ FILE: .readthedocs.yaml ================================================ # .readthedocs.yaml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details # Required version: 2 # Set the OS, Python version and other tools you might need build: os: ubuntu-22.04 tools: python: "3.10" jobs: post_create_environment: - pip install poetry - poetry config virtualenvs.create false post_install: - poetry install - pip install -e . # Build documentation in the "docs/" directory with Sphinx sphinx: configuration: docs/source/conf.py # Optional but recommended, declare the Python requirements required # to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - requirements: docs/requirements.txt ================================================ FILE: .vscode/tasks.json ================================================ { // See https://go.microsoft.com/fwlink/?LinkId=733558 // for the documentation about the tasks.json format "version": "2.0.0", "tasks": [ { "label": "Doc builder", "type": "shell", "command": "source ${workspaceFolder}/.venv/bin/activate && find source/ | entr -s 'make clean && make html'", "options": { "cwd": "${workspaceFolder}/docs" }, "problemMatcher": [], "group": { "kind": "build" }, "isBackground": true, "presentation": { "echo": true, "reveal": "always", "focus": true, "panel": "dedicated", "showReuseMessage": false, "clear": true, "close": true } } ] } ================================================ FILE: CHANGELOG.md ================================================ # Changelog ## 2.2.2 - 2/23/25 - Bugfix where it was impossible to use a signal as an `ok_code` [#699](https://github.com/amoffat/sh/issues/699) ## 2.2.1 - 1/9/25 - Bugfix where `async` and `return_cmd` does not raise exceptions [#746](https://github.com/amoffat/sh/pull/746) ## 2.2.0 - 1/9/25 - `return_cmd` with `await` now works correctly [#743](https://github.com/amoffat/sh/issues/743) - Formal support for Python 3.12 ## 2.1.0 - 10/8/24 - Add contrib command `sh.contrib.bash` [#736](https://github.com/amoffat/sh/pull/736) ## 2.0.7 - 5/31/24 - Fix `sh.glob` arguments [#708](https://github.com/amoffat/sh/issues/708) - Misc modernizations ## 2.0.6 - 8/9/23 - Add back appropriate sdist files [comment](https://github.com/amoffat/sh/commit/89333ae48069a5b445b3535232195b2de6f4648f) ## 2.0.5 - 8/7/23 - Allow nested `with` contexts [#690](https://github.com/amoffat/sh/issues/690) - Call correct asyncio function for getting event loop [#683](https://github.com/amoffat/sh/issues/683) ## 2.0.4 - 5/13/22 - Allow `ok_code` to be used with `fg` [#665](https://github.com/amoffat/sh/pull/665) - Make sure `new_group` never creates a new session [#675](https://github.com/amoffat/sh/pull/675) ## 2.0.2 / 2.0.3 (misversioned) - 2/13/22 - Performance regression when using a generator with `_in` [#650](https://github.com/amoffat/sh/pull/650) - Adding test support for python 3.11 ## 2.0.0 - 2/9/22 - Executed commands now return a unicode string by default - Removed magical module-like execution contexts [#636](https://github.com/amoffat/sh/issues/636) - Added basic asyncio support via `_async` - Dropped support for Python < 3.8 - Bumped default tty size to more standard (24, 80) - First argument being a RunningCommand no longer automatically passes it as stdin - `RunningCommand.__eq__` no longer has the side effect of executing the command [#518](https://github.com/amoffat/sh/pull/531) - `_tee` now supports both "err" and "out" [#215](https://github.com/amoffat/sh/issues/215) - Removed the builtin override `cd` [link](https://github.com/amoffat/sh/pull/584#discussion_r698055681) - Altered process launching model to behave more expectedly [#495](https://github.com/amoffat/sh/issues/495) - Bugfix where `_no_out` isn't allowed with `_iter="err"` [#638](https://github.com/amoffat/sh/issues/638) - Allow keyword arguments to have a list of values [#529](https://github.com/amoffat/sh/issues/529) ## 1.14.3 - 7/17/22 - Bugfix where `Command` was not aware of default call args when wrapping the module [#559](https://github.com/amoffat/sh/pull/573) ## 1.14.1 - 10/24/20 - bugfix where setting `_ok_code` to not include 0, but 0 was the exit code [#545](https://github.com/amoffat/sh/pull/545) ## 1.14.0 - 8/28/20 - `_env` now more lenient in accepting dictionary-like objects [#527](https://github.com/amoffat/sh/issues/527) - `None` and `False` arguments now do not pass through to underlying command [#525](https://github.com/amoffat/sh/pull/525) - Implemented `find_spec` on the fancy importer, which fixes some Python3.4+ issues [#536](https://github.com/amoffat/sh/pull/536) ## 1.13.1 - 4/28/20 - regression fix if `_fg=False` [#520](https://github.com/amoffat/sh/issues/520) ## 1.13.0 - 4/27/20 - minor Travis CI fixes [#492](https://github.com/amoffat/sh/pull/492) - bugfix for boolean long options not respecting `_long_prefix` [#488](https://github.com/amoffat/sh/pull/488) - fix deprecation warning on Python 3.6 regexes [#482](https://github.com/amoffat/sh/pull/482) - `_pass_fds` and `_close_fds` special kwargs for controlling file descriptor inheritance in child. - more efficiently closing inherited fds [#406](https://github.com/amoffat/sh/issues/406) - bugfix where passing invalid dictionary to `_env` will cause a mysterious child 255 exit code. [#497](https://github.com/amoffat/sh/pull/497) - bugfix where `_in` using 0 or `sys.stdin` wasn't behaving like a TTY, if it was in fact a TTY. [#514](https://github.com/amoffat/sh/issues/514) - bugfix where `help(sh)` raised an exception [#455](https://github.com/amoffat/sh/issues/455) - bugfix fixing broken interactive ssh tutorial from docs - change to automatic tty merging into a single pty if `_tty_in=True` and `_tty_out=True` - introducing `_unify_ttys`, default False, which allows explicit tty merging into single pty - contrib command for `ssh` connections requiring passwords - performance fix for polling output too fast when using `_iter` [#462](https://github.com/amoffat/sh/issues/462) - execution contexts can now be used in python shell [#466](https://github.com/amoffat/sh/pull/466) - bugfix `ErrorReturnCode` instances can now be pickled - bugfix passing empty string or `None` for `_in` hanged [#427](https://github.com/amoffat/sh/pull/427) - bugfix where passing a filename or file-like object to `_out` wasn't using os.dup2 [#449](https://github.com/amoffat/sh/issues/449) - regression make `_fg` work with `_cwd` again [#330](https://github.com/amoffat/sh/issues/330) - an invalid `_cwd` now raises a `ForkException` not an `OSError`. - AIX support [#477](https://github.com/amoffat/sh/issues/477) - added a `timeout=None` param to `RunningCommand.wait()` [#515](https://github.com/amoffat/sh/issues/515) ## 1.12.14 - 6/6/17 - bugfix for poor sleep performance [#378](https://github.com/amoffat/sh/issues/378) - allow passing raw integer file descriptors for `_out` and `_err` handlers - bugfix for when `_tee` and `_out` are used, and the `_out` is a tty or pipe [#384](https://github.com/amoffat/sh/issues/384) - bugfix where python 3.3+ detected different arg counts for bound method output callbacks [#380](https://github.com/amoffat/sh/issues/380) ## 1.12.12, 1.12.13 - 3/30/17 - pypi readme doc bugfix [PR#377](https://github.com/amoffat/sh/pull/377) ## 1.12.11 - 3/13/17 - bugfix for relative paths to `sh.Command` not expanding to absolute paths [#372](https://github.com/amoffat/sh/issues/372) - updated for python 3.6 - bugfix for SIGPIPE not being handled correctly on pipelined processes [#373](https://github.com/amoffat/sh/issues/373) ## 1.12.10 - 3/02/17 - bugfix for file descriptors over 1024 [#356](https://github.com/amoffat/sh/issues/356) - bugfix when `_err_to_out` is True and `_out` is pipe or tty [#365](https://github.com/amoffat/sh/issues/365) ## 1.12.9 - 1/04/17 - added `_bg_exc` for silencing exceptions in background threads [#350](https://github.com/amoffat/sh/pull/350) ## 1.12.8 - 12/16/16 - bugfix for patched glob.glob on python3.5 [#341](https://github.com/amoffat/sh/issues/341) ## 1.12.7 - 12/07/16 - added `_out` and `_out_bufsize` validator [#346](https://github.com/amoffat/sh/issues/346) - bugfix for internal stdout thread running when it shouldn't [#346](https://github.com/amoffat/sh/issues/346) ## 1.12.6 - 12/02/16 - regression bugfix on timeout [#344](https://github.com/amoffat/sh/issues/344) - regression bugfix on `_ok_code=None` ## 1.12.5 - 12/01/16 - further improvements on cpu usage ## 1.12.4 - 11/30/16 - regression in cpu usage [#339](https://github.com/amoffat/sh/issues/339) ## 1.12.3 - 11/29/16 - fd leak regression and fix for flawed fd leak detection test [#337](https://github.com/amoffat/sh/pull/337) ## 1.12.2 - 11/28/16 - support for `io.StringIO` in python2 ## 1.12.1 - 11/28/16 - added support for using raw file descriptors for `_in`, `_out`, and `_err` - removed `.close()`ing `_out` handler if FIFO detected ## 1.12.0 - 11/21/16 - composed commands no longer propagate `_bg` - better support for using `sys.stdin` and `sys.stdout` for `_in` and `_out` - bugfix where `which()` would not stop searching at the first valid executable found in PATH - added `_long_prefix` for programs whose long arguments start with something other than `--` [#278](https://github.com/amoffat/sh/pull/278) - added `_log_msg` for advanced configuration of log message [#311](https://github.com/amoffat/sh/pull/311) - added `sh.contrib.sudo` - added `_arg_preprocess` for advanced command wrapping - alter callable `_in` arguments to signify completion with falsy chunk - bugfix where pipes passed into `_out` or `_err` were not flushed on process end [#252](https://github.com/amoffat/sh/pull/252) - deprecated `with sh.args(**kwargs)` in favor of `sh2 = sh(**kwargs)` - made `sh.pushd` thread safe - added `.kill_group()` and `.signal_group()` methods for better process control [#237](https://github.com/amoffat/sh/pull/237) - added `new_session` special keyword argument for controlling spawned process session [#266](https://github.com/amoffat/sh/issues/266) - bugfix better handling for EINTR on system calls [#292](https://github.com/amoffat/sh/pull/292) - bugfix where with-contexts were not threadsafe [#247](https://github.com/amoffat/sh/issues/195) - `_uid` new special keyword param for specifying the user id of the process [#133](https://github.com/amoffat/sh/issues/133) - bugfix where exceptions were swallowed by processes that weren't waited on [#309](https://github.com/amoffat/sh/issues/309) - bugfix where processes that dupd their stdout/stderr to a long running child process would cause sh to hang [#310](https://github.com/amoffat/sh/issues/310) - improved logging output [#323](https://github.com/amoffat/sh/issues/323) - bugfix for python3+ where binary data was passed into a process's stdin [#325](https://github.com/amoffat/sh/issues/325) - Introduced execution contexts which allow baking of common special keyword arguments into all commands [#269](https://github.com/amoffat/sh/issues/269) - `Command` and `which` now can take an optional `paths` parameter which specifies the search paths [#226](https://github.com/amoffat/sh/issues/226) - `_preexec_fn` option for executing a function after the child process forks but before it execs [#260](https://github.com/amoffat/sh/issues/260) - `_fg` reintroduced, with limited functionality. hurrah! [#92](https://github.com/amoffat/sh/issues/92) - bugfix where a command would block if passed a fd for stdin that wasn't yet ready to read [#253](https://github.com/amoffat/sh/issues/253) - `_long_sep` can now take `None` which splits the long form arguments into individual arguments [#258](https://github.com/amoffat/sh/issues/258) - making `_piped` perform "direct" piping by default (linking fds together). this fixes memory problems [#270](https://github.com/amoffat/sh/issues/270) - bugfix where calling `next()` on an iterable process that has raised `StopIteration`, hangs [#273](https://github.com/amoffat/sh/issues/273) - `sh.cd` called with no arguments no changes into the user's home directory, like native `cd` [#275](https://github.com/amoffat/sh/issues/275) - `sh.glob` removed entirely. the rationale is correctness over hand-holding. [#279](https://github.com/amoffat/sh/issues/279) - added `_truncate_exc`, defaulting to `True`, which tells our exceptions to truncate output. - bugfix for exceptions whose messages contained unicode - `_done` callback no longer assumes you want your command put in the background. - `_done` callback is now called asynchronously in a separate thread. - `_done` callback is called regardless of exception, which is necessary in order to release held resources, for example a process pool ## 1.10 - 12/30/14 - partially applied functions with `functools.partial` have been fixed for `_out` and `_err` callbacks [#160](https://github.com/amoffat/sh/issues/160) - `_out` or `_err` being callables no longer puts the running command in the background. to achieve the previous behavior, pass `_bg=True` to your command. - deprecated `_with` contexts [#195](https://github.com/amoffat/sh/issues/195) - `_timeout_signal` allows you to specify your own signal to kill a timed-out process with. use a constant from the `signal` stdlib module. [#171](https://github.com/amoffat/sh/issues/171) - signal exceptions can now be caught by number or name. `SignalException_9 == SignalException_SIGKILL` - child processes that timeout via `_timeout` raise `sh.TimeoutException` instead of `sh.SignalExeception_9` [#172](https://github.com/amoffat/sh/issues/172) - fixed `help(sh)` from the python shell and `pydoc sh` from the command line. [#173](https://github.com/amoffat/sh/issues/173) - program names can no longer be shadowed by names that sh.py defines internally. removed the requirement of trailing underscores for programs that could have their names shadowed, like `id`. - memory optimization when a child process's stdin is a newline-delimted string and our bufsize is newlines - feature, `_done` special keyword argument that accepts a callback to be called when the command completes successfully [#185](https://github.com/amoffat/sh/issues/185) - bugfix for being unable to print a baked command in python3+ [#176](https://github.com/amoffat/sh/issues/176) - bugfix for cwd not existing and causing the child process to continue running parent process code [#202](https://github.com/amoffat/sh/issues/202) - child process is now guaranteed to exit on exception between fork and exec. - fix python2 deprecation warning when running with -3 [PR #165](https://github.com/amoffat/sh/pull/165) - bugfix where sh.py was attempting to execute directories [#196](https://github.com/amoffat/sh/issues/196), [PR #189](https://github.com/amoffat/sh/pull/189) - only backgrounded processes will ignore SIGHUP - allowed `ok_code` to take a `range` object. [#PR 210](https://github.com/amoffat/sh/pull/210/files) - added `sh.args` with context which allows overriding of all command defaults for the duration of that context. - added `sh.pushd` with context which takes a directory name and changes to that directory for the duration of that with context. [PR #206](https://github.com/amoffat/sh/pull/206) - tests now include python 3.4 if available. tests also stop on the first python that suite that fails. - SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGPIPE, SIGSYS have been added to the list of signals that throw an exception [PR #201](https://github.com/amoffat/sh/pull/201) - "callable" builtin has been faked for python3.1, which lacks it. - "direct" option added to `_piped` special keyword argument, which allows sh to hand off a process's stdout fd directly to another process, instead of buffering its stdout internally, then handing it off. [#119](https://github.com/amoffat/sh/issues/119) ## 1.09 - 9/08/13 - Fixed encoding errors related to a system encoding "ascii". [#123](https://github.com/amoffat/sh/issues/123) - Added exit_code attribute to SignalException and ErrorReturnCode exception classes. [#127](https://github.com/amoffat/sh/issues/127) - Making the default behavior of spawned processes to not be explicitly killed when the parent python process ends. Also making the spawned process ignore SIGHUP. [#139](https://github.com/amoffat/sh/issues/139) - Made OSX sleep hack to apply to PY2 as well as PY3. ## 1.08 - 1/29/12 - Added SignalException class and made all commands that end terminate by a signal defined in SIGNALS_THAT_SHOULD_THROW_EXCEPTION raise it. [#91](https://github.com/amoffat/sh/issues/91) - Bugfix where CommandNotFound was not being raised if Command was created by instantiation. [#113](https://github.com/amoffat/sh/issues/113) - Bugfix for Commands that are wrapped with functools.wraps() [#121](https://github.com/amoffat/sh/issues/121] - Bugfix where input arguments were being assumed as ascii or unicode, but never as a string in a different encoding. - \_long_sep keyword argument added joining together a dictionary of arguments passed in to a command - Commands can now be passed a dictionary of args, and the keys will be interpretted "raw", with no underscore-to-hyphen conversion - Reserved Python keywords can now be used as subcommands by appending an underscore `_` to them ## 1.07 - 11/21/12 - Bugfix for PyDev when `locale.getpreferredencoding()` is empty. - Fixes for IPython3 that involve `sh.` and `sh?` - Added `_tee` special keyword argument to force stdout/stderr to store internally and make available for piping data that is being redirected. - Added `_decode_errors` to be passed to all stdout/stderr decoding of a process. - Added `_no_out`, `_no_err`, and `_no_pipe` special keyword arguments. These are used for long-running processes with lots of output. - Changed custom loggers that were created for each process to fixed loggers, so there are no longer logger references laying around in the logging module after the process ends and it garbage collected. ## 1.06 - 11/10/12 - Removed old undocumented cruft of ARG1..ARGN and ARGV. - Bugfix where `logging_enabled` could not be set from the importing module. - Disabled garbage collection before fork to prevent garbage collection in child process. - Major bugfix where cyclical references were preventing process objects (and their associated stdout/stderr buffers) from being garbage collected. - Bugfix in RunningCommand and OProc loggers, which could get really huge if a command was called that had a large number of arguments. ## 1.05 - 10/20/12 - Changing status from alpha to beta. - Python 3.3 officially supported. - Documentation fix. The section on exceptions now references the fact that signals do not raise an exception, even for signals that might seem like they should, e.g. segfault. - Bugfix with Python 3.3 where importing commands from the sh namespace resulted in an error related to `__path__` - Long-form and short-form options to commands may now be given False to disable the option from being passed into the command. This is useful to pass in a boolean flag that you flip to either True or False to enable or disable some functionality at runtime. ## 1.04 - 10/07/12 - Making `Command` class resolve the `path` parameter with `which` by default instead of expecting it to be resolved before it is passed in. This change shouldn't affect backwards compatibility. - Fixing a bug when an exception is raised from a program, and the error output has non-ascii text. This didn't work in Python < 3.0, because .decode()'s default encoding is typically ascii. ================================================ FILE: CODEOWNERS ================================================ /.github/ @amoffat ================================================ FILE: LICENSE.txt ================================================ Copyright (C) 2011-2012 by Andrew Moffat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MIGRATION.md ================================================ # Migrating from 1._ to 2._ This document provides an upgrade path from `1.*` to `2.*`. ## `sh.cd` builtin removed There is no `sh.cd` command anymore. It was always command implemented in sh, as some systems provide it as a shell builtin, while others have an actual binary. But neither of them persisted the directory change between other `sh` calls, which is why it was implemented in sh. ### Workaround If you were using `sh.cd(dir)`, use the context manager `with sh.pushd(dir)` instead. All of the commands in the managed context will have the correct directory. ## Removed execution contexts / default arguments In `1.*` you could do could spawn a new module from the `sh` module, one which had customized defaults for the special keyword arguments. This module could then be accessed just like `sh`, and you could even import commands from it. Unfortunately the magic required to make that work was brittle. Also it was not aligned syntactically with the similar baking concept. We have therefore changed the syntax to align with baking, and also removed the ability to import directly from this new baked execution context. ### Workaround ```python sh2 = sh(_tty_out=False) sh2.ls() ``` Becomes: ```python sh2 = sh.bake(_tty_out=False) sh2.ls() ``` And ```python sh2 = sh.bake(_tty_out=False) from sh2 import ls ls() ``` Becomes: ```python sh2 = sh.bake(_tty_out=False) ls = sh2.ls ls() ``` ## Return value now a true string In `2.*`, the return value of an executed `sh` command has changed (in most cases) from a `RunningCommand` object to a unicode string. This makes using the output of a command more natural. ### Workaround To continue returning a `RunningCommand` object, you must use the `_return_cmd=True` special keyword argument. You can achieve this on each file with the following code at the top of files that use `sh`: ```python import sh sh = sh.bake(_return_cmd=True) ``` ## Piping to STDIN Previously, if the first argument of a sh command was an instance of `RunningCommand`, it was automatically fed into the process's STDIN. This is no longer the case and you must explicitly use `_in=`. ```python from sh import wc,ls print(wc(ls("/home/", "-l"), "-l")) ``` Becomes: ```python from sh import wc,ls print(wc("-l", _in=ls("/home/", "-l"))) ``` Or: ```python from sh import wc,ls print(wc("-l", _in=ls("/home/", "-l", _return_cmd=True))) ``` ### Workaround None ## New processes don't launch in new session In `1.*`, `_new_session` defaulted to `True`. It now defaults to `False`. The reason for this is that it makes more sense for launched processes to default to being in the process group of the python script, so that they receive SIGINTs correctly. ### Workaround To preserve the old behavior: ```python import sh sh = sh.bake(_new_session=True) ``` ================================================ FILE: Makefile ================================================ # runs all tests on all envs, in parallel .PHONY: test test: build_test_image docker run -it --rm amoffat/shtest tox -p # one test on all envs, in parallel .PHONY: test_one test_one: build_test_image docker run -it --rm amoffat/shtest tox -p -- $(test) .PHONY: build_test_image build_test_image: docker build -t amoffat/shtest -f tests/Dockerfile --build-arg cache_bust=951 . # publishes to PYPI .PHONY: release release: poetry publish --dry-run ================================================ FILE: README.rst ================================================ .. image:: https://raw.githubusercontent.com/amoffat/sh/master/images/logo-230.png :target: https://amoffat.github.com/sh :alt: Logo **If you are migrating from 1.* to 2.*, please see MIGRATION.md** | .. image:: https://img.shields.io/pypi/v/sh.svg?style=flat-square :target: https://pypi.python.org/pypi/sh :alt: Version .. image:: https://img.shields.io/pypi/dm/sh.svg?style=flat-square :target: https://pypi.python.org/pypi/sh :alt: Downloads Status .. image:: https://img.shields.io/pypi/pyversions/sh.svg?style=flat-square :target: https://pypi.python.org/pypi/sh :alt: Python Versions .. image:: https://img.shields.io/coveralls/amoffat/sh.svg?style=flat-square :target: https://coveralls.io/r/amoffat/sh?branch=master :alt: Coverage Status | sh is a full-fledged subprocess replacement for Python 3.8 - 3.12, and PyPy that allows you to call *any* program as if it were a function: .. code:: python from sh import ifconfig print(ifconfig("eth0")) sh is *not* a collection of system commands implemented in Python. sh relies on various Unix system calls and only works on Unix-like operating systems - Linux, macOS, BSDs etc. Specifically, Windows is not supported. `Complete documentation here `_ `Full documentation on a single page for LLM-assisted coding here `_ Installation ============ :: $> pip install sh Support ======= * `Andrew Moffat `_ - author/maintainer * `Erik Cederstrand `_ - maintainer Developers ========== Testing ------- Tests are run in a docker container against all supported Python versions. To run, make the following target:: $> make test To run a single test:: $> make test='FunctionalTests.test_background' test_one Docs ---- To build the docs, make sure you've run ``poetry install`` to install the dev dependencies, then:: $> cd docs $> make html This will generate the docs in ``docs/build/html``. You can open the ``index.html`` file in your browser to view the docs. Coverage -------- First run all of the tests:: $> SH_TESTS_RUNNING=1 coverage run --source=sh -m pytest This will aggregate a ``.coverage``. You may then visualize the report with:: $> coverage report Or generate visual html files with:: $> coverage html Which will create ``./htmlcov/index.html`` that you may open in a web browser. ================================================ FILE: dev_scripts/changelog_extract.py ================================================ import re import sys from pathlib import Path from typing import Iterable THIS_DIR = Path(__file__).parent CHANGELOG = THIS_DIR.parent / "CHANGELOG.md" def fetch_changes(changelog: Path, version: str) -> Iterable[str]: with open(changelog, "r") as f: lines = f.readlines() found_a_change = False aggregate = False for line in lines: if line.startswith(f"## {version}"): aggregate = True if aggregate: if line.startswith("-"): line = re.sub(r"-\s*", "", line).strip() found_a_change = True yield line elif found_a_change: aggregate = False return changes version = sys.argv[1].strip() changes = fetch_changes(CHANGELOG, version) if not changes: exit(1) for change in changes: print("- " + change) ================================================ FILE: docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= -a -W SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/requirements.txt ================================================ toml==0.10.2 ================================================ FILE: docs/source/conf.py ================================================ # Configuration file for the Sphinx documentation builder. from pathlib import Path import toml _THIS_DIR = Path(__file__).parent _REPO = _THIS_DIR.parent.parent _PYPROJECT = _REPO / "pyproject.toml" pyproject = toml.load(_PYPROJECT) nitpicky = True nitpick_ignore = [ ("py:class", "Token"), ("py:class", "'Token'"), ] nitpick_ignore_regex = [ ("py:class", r".*lark.*"), ] version = pyproject["tool"]["poetry"]["version"] release = version # -- Project information project = "sh" copyright = "2023, Andrew Moffat" author = "Andrew Moffat" # -- General configuration extensions = [ "sphinx.ext.duration", "sphinx.ext.doctest", "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.intersphinx", "sphinx.ext.todo", ] intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), "sphinx": ("https://www.sphinx-doc.org/en/master/", None), "lark": ("https://lark-parser.readthedocs.io/en/latest/", None), "jinja2": ("https://jinja.palletsprojects.com/en/latest/", None), } intersphinx_disabled_domains = ["std"] templates_path = ["_templates"] # -- Options for HTML output html_theme = "sphinx_rtd_theme" # -- Options for EPUB output epub_show_urls = "footnote" autodoc_typehints = "both" add_module_names = False ================================================ FILE: docs/source/examples/done.rst ================================================ Here's an example of using :ref:`done` to create a multiprocess pool, where ``sh.your_parallel_command`` is executed concurrently at no more than 10 at a time: .. code-block:: python import sh from threading import Semaphore pool = Semaphore(10) def done(cmd, success, exit_code): pool.release() def do_thing(arg): pool.acquire() return sh.your_parallel_command(arg, _bg=True, _done=done) procs = [] for arg in range(100): procs.append(do_thing(arg)) # essentially a join [p.wait() for p in procs] ================================================ FILE: docs/source/fulldoc.rst ================================================ Full Documentation ================== This single page repeats the full documentation for `sh `, making it easier to put into an LLM's context window. There is nothing on this page that is not mentionned already elsewhere on this site, it's just reorganized as a single page. .. include:: index.rst .. include:: tutorials/interacting_with_processes.rst .. include:: tutorials/real_time_output.rst .. content linked in index.rst .. include:: sections/faq.rst .. include:: sections/contrib.rst .. include:: sections/sudo.rst .. content of usage.rst .. include:: sections/passing_arguments.rst .. include:: sections/exit_codes.rst .. include:: sections/redirection.rst .. include:: sections/asynchronous_execution.rst .. also contains reference to example/done.rst so no need to mention it explicitely .. .. include:: examples/done.rst .. include:: sections/baking.rst .. include:: sections/piping.rst .. include:: sections/subcommands.rst .. include:: sections/default_arguments.rst .. include:: sections/envs.rst .. include:: sections/stdin.rst .. include:: sections/with.rst .. content of reference.rst .. include:: sections/special_arguments.rst .. include:: sections/architecture.rst .. include:: sections/command_class.rst ================================================ FILE: docs/source/index.rst ================================================ .. toctree:: :hidden: usage reference sections/contrib sections/sudo tutorials sections/faq ref_to_fulldoc .. image:: images/logo-230.png :alt: Logo sh ## .. image:: https://img.shields.io/pypi/v/sh.svg?style=flat-square :target: https://pypi.python.org/pypi/sh :alt: Version .. image:: https://img.shields.io/pypi/dm/sh.svg?style=flat-square :target: https://pypi.python.org/pypi/sh :alt: Downloads Status .. image:: https://img.shields.io/pypi/pyversions/sh.svg?style=flat-square :target: https://pypi.python.org/pypi/sh :alt: Python Versions .. image:: https://img.shields.io/coveralls/amoffat/sh.svg?style=flat-square :target: https://coveralls.io/r/amoffat/sh?branch=master :alt: Coverage Status .. image:: https://img.shields.io/github/stars/amoffat/sh.svg?style=social&label=Star :target: https://github.com/amoffat/sh :alt: Github sh is a full-fledged subprocess replacement for Python 3.8+, and PyPy that allows you to call any program as if it were a function: .. code-block:: python from sh import ifconfig print(ifconfig("wlan0")) Output: .. code-block:: none wlan0 Link encap:Ethernet HWaddr 00:00:00:00:00:00 inet addr:192.168.1.100 Bcast:192.168.1.255 Mask:255.255.255.0 inet6 addr: ffff::ffff:ffff:ffff:fff/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000 RX bytes:0 (0 GB) TX bytes:0 (0 GB) Note that these aren't Python functions, these are running the binary commands on your system by dynamically resolving your ``$PATH``, much like Bash does, and then wrapping the binary in a function. In this way, all the programs on your system are easily available to you from within Python. sh relies on various Unix system calls and only works on Unix-like operating systems - Linux, macOS, BSDs etc. Specifically, Windows is not supported. Installation ============ .. code-block:: none pip install sh Quick Reference =============== Passing Arguments ----------------- .. code-block:: python sh.ls("-l", "/tmp", color="never") :ref:`Read More ` Exit Codes ---------- .. code-block:: python try: sh.ls("/doesnt/exist") except sh.ErrorReturnCode_2: print("directory doesn't exist") :ref:`Read More ` Redirection ----------- .. code-block:: python sh.ls(_out="/tmp/dir_contents") with open("/tmp/dir_contents", "w") as h: sh.ls(_out=h) from io import StringIO buf = StringIO() sh.ls(_out=buf) :ref:`Read More ` Baking ------ .. code-block:: python my_ls = sh.ls.bake("-l") # equivalent my_ls("/tmp") sh.ls("-l", "/tmp") :ref:`Read More ` Piping ------ .. code-block:: python sh.wc("-l", _in=sh.ls("-1")) :ref:`Read More ` Subcommands ----------- .. code-block:: python # equivalent sh.git("show", "HEAD") sh.git.show("HEAD") :ref:`Read More ` Background Processes -------------------- .. code-block:: python p = sh.find("-name", "sh.py", _bg=True) # ... do other things ... p.wait() :ref:`Read More ` .. include:: ref_to_fulldoc.rst ================================================ FILE: docs/source/ref_to_fulldoc.rst ================================================ Single Page =========== The page below repeats the full documentation for `sh ` as a single page, making it easier to put into an LLM's context window. :doc:`./fulldoc` ================================================ FILE: docs/source/reference.rst ================================================ Reference ========= .. toctree:: sections/special_arguments sections/architecture sections/command_class ================================================ FILE: docs/source/sections/architecture.rst ================================================ .. _architecture: Architecture Overview ##################### Launch ====== When it comes time to launch a process #. Open pipes and/or TTYs STDIN/OUT/ERR. #. Open a pipe for communicating pre-exec exceptions from the child to the parent. #. Open a pipe for child/parent launch synchronization. #. :func:`os.fork` a child process. From here, we have two concurrent processes running: Child ----- #. If :ref:`_bg=True ` is set, we ignore :py:data:`signal.SIGHUP`. #. If :ref:`_new_session=True `, become a session leader with :func:`os.setsid`, else become a process group leader with :func:`os.setpgrp`. #. Write our session id to the a pipe connected to the parent. This is mainly to synchronize with our parent that our session/group logic has finished. #. :func:`os.dup2` the file descriptors of our previously-setup TTYs/pipes to our STDIN/OUT/ERR file descriptors. #. If we're a session leader and our STDIN is a TTY, via :ref:`_tty_in=True `, acquire a controlling terminal, thereby becoming the controlling process of the session. #. Set our GID/UID if we've set a custom one via :ref:`_uid `. #. Close all file descriptors greater than STDERR. #. Call :func:`os.execv`. Parent ------ #. Check for any exceptions via the exception pipe connected to the child. #. Block and read our child's session id from a pipe connected to the child. This synchronizes to us that the child has finished moving between sessions/groups and we can now accurately determine its current session id and process group. #. If we're using a TTY for STDIN, via :ref:`_tty_in=True `, disable echoing on the TTY, so that data sent to STDIN is not echoed to STDOUT. Running ======= An instance of :ref:`oproc_class` contains two internal threads, one for STDIN, and one for STDOUT and STDERR. The purpose of these threads is to handle reading/writing to the read/write ends of the process's standard descriptors. For example, the STDOUT/ERR thread continually runs :func:`select.select` on the master ends of the TTYs/pipes connected to STDOUT/ERR, and if they're ready to read, reads the available data and aggregates it into the appropriate place. .. _arch_buffers: Buffers ------- A couple of different buffers must be considered when thinking about how data flows through an sh process. The first buffer is the buffer associated with the underlying pipe or TTY attached to STDOUT/ERR. In the case of a TTY (the default for output), the buffer size is 0, so output is immediate -- a byte written by the process is a byte received by sh. For a pipe, however, the buffer size of the pipe is typically 4-64kb. :manpage:`pipe(2)`. .. seealso:: FAQ: :ref:`faq_tty_out` The second buffer is sh's internal buffers, one for STDOUT and one for STDERR. These buffers aggregate data that has been read from the master end of the TTY or pipe attached to the output fd, but before that data is sent along to the appropriate output handler (queue, file object, function, etc). Data sits in these buffers until we reach the size specified with :ref:`internal_bufsize`, at which point the buffer flushes to the output handler. Exit ==== STDIN Thread Shutdown --------------------- On process completion, our internal threads must complete, as the read end of STDIN, for example, which is connected to the process, is no longer open, so writing to the slave end will no longer work. STDOUT/ERR Thread Shutdown -------------------------- The STDOUT/ERR thread is a little more complicated, because although the process is not alive, output data may still exist in the pipe/TTY buffer that must be collected. So we essentially just :func:`select.select` on the read ends until they return nothing, indicating that they are complete, then we break out of our read loop. .. _arch_exit_code: Exit Code Processing -------------------- The exit code is obtained from the reaped process. If the process ended from a signal, the exit code is the negative value of that signal. For example, SIGKILL would result in an exit code -9. Done Callback ------------- If specified, the :ref:`done` callback is executed with the :ref:`RunningCommand ` instance, a boolean indicating success, and the adjusted exit code. After the callback returns, error processing continues. In other words, the done callback is called regardless of success or failure, and there's nothing it can do to prevent the :ref:`ErrorReturnCode ` exceptions from being raised after it completes. ================================================ FILE: docs/source/sections/asynchronous_execution.rst ================================================ .. _async: Asynchronous Execution ###################### sh provides a few methods for running commands and obtaining output in a non-blocking fashion. AsyncIO ======= .. versionadded:: 2.0.0 Sh supports asyncio on commands with the :ref:`_async=True ` special kwarg. This let's you incrementally ``await`` output produced from your command. .. code-block:: python import asyncio import sh async def main(): await sh.sleep(3, _async=True) asyncio.run(main()) .. _iterable: Incremental Iteration ===================== You may also create asynchronous commands by iterating over them with the :ref:`iter` special kwarg. This creates an iterable (specifically, a generator) that you can loop over: .. code-block:: python from sh import tail # runs forever for line in tail("-f", "/var/log/some_log_file.log", _iter=True): print(line) By default, :ref:`iter` iterates over STDOUT, but you can change set this specifically by passing either ``"err"`` or ``"out"`` to :ref:`iter` (instead of ``True``). Also by default, output is line-buffered, so the body of the loop will only run when your process produces a newline. You can change this by changing the buffer size of the command's output with :ref:`out_bufsize`. .. note:: If you need a *fully* non-blocking iterator, use :ref:`iter_noblock`. If the current iteration would block, :py:data:`errno.EWOULDBLOCK` will be returned, otherwise you'll receive a chunk of output, as normal. .. _background: Background Processes ==================== By default, each running command blocks until completion. If you have a long-running command, you can put it in the background with the :ref:`_bg=True ` special kwarg: .. code-block:: python # blocks sleep(3) print("...3 seconds later") # doesn't block p = sleep(3, _bg=True) print("prints immediately!") p.wait() print("...and 3 seconds later") You'll notice that you need to call :meth:`RunningCommand.wait` in order to exit after your command exits. Commands launched in the background ignore ``SIGHUP``, meaning that when their controlling process (the session leader, if there is a controlling terminal) exits, they will not be signalled by the kernel. But because sh commands launch their processes in their own sessions by default, meaning they are their own session leaders, ignoring ``SIGHUP`` will normally have no impact. So the only time ignoring ``SIGHUP`` will do anything is if you use :ref:`_new_session=False `, in which case the controlling process will probably be the shell from which you launched python, and exiting that shell would normally send a ``SIGHUP`` to all child processes. .. seealso:: For more information on the exact launch process, see :ref:`architecture`. .. _callbacks: Output Callbacks ---------------- In combination with :ref:`_bg=True`, sh can use callbacks to process output incrementally by passing a callable function to :ref:`out` and/or :ref:`err`. This callable will be called for each line (or chunk) of data that your command outputs: .. code-block:: python from sh import tail def process_output(line): print(line) p = tail("-f", "/var/log/some_log_file.log", _out=process_output, _bg=True) p.wait() To control whether the callback receives a line or a chunk, use :ref:`out_bufsize`. To "quit" your callback, simply return ``True``. This tells the command not to call your callback anymore. The line or chunk received by the callback can either be of type ``str`` or ``bytes``. If the output could be decoded using the provided encoding, a ``str`` will be passed to the callback, otherwise it would be raw ``bytes``. .. note:: Returning ``True`` does not kill the process, it only keeps the callback from being called again. See :ref:`interactive_callbacks` for how to kill a process from a callback. .. seealso:: :ref:`red_func` .. _interactive_callbacks: Interactive callbacks --------------------- Commands may communicate with the underlying process interactively through a specific callback signature Each command launched through sh has an internal STDIN :class:`queue.Queue` that can be used from callbacks: .. code-block:: python def interact(line, stdin): if line == "What... is the air-speed velocity of an unladen swallow?": stdin.put("What do you mean? An African or European swallow?") elif line == "Huh? I... I don't know that....AAAAGHHHHHH": cross_bridge() return True else: stdin.put("I don't know....AAGGHHHHH") return True p = sh.bridgekeeper(_out=interact, _bg=True) p.wait() .. note:: If you use a queue, you can signal the end of the input (EOF) with ``None`` You can also kill or terminate your process (or send any signal, really) from your callback by adding a third argument to receive the process object: .. code-block:: python def process_output(line, stdin, process): print(line) if "ERROR" in line: process.kill() return True p = tail("-f", "/var/log/some_log_file.log", _out=process_output, _bg=True) The above code will run, printing lines from ``some_log_file.log`` until the word ``"ERROR"`` appears in a line, at which point the tail process will be killed and the script will end. .. note:: You may also use :meth:`RunningCommand.terminate` to send a SIGTERM, or :meth:`RunningCommand.signal` to send a general signal. Done Callbacks -------------- A done callback called when the process exits, either normally (through a success or error exit code) or through a signal. It is *always* called. .. include:: /examples/done.rst ================================================ FILE: docs/source/sections/baking.rst ================================================ .. _baking: Baking ====== sh is capable of "baking" arguments into commands. This is essentially `partial application `_, like you might do with :func:`functools.partial`. .. code-block:: python from sh import ls ls = ls.bake("-la") print(ls) # "/usr/bin/ls -la" # resolves to "ls -la /" print(ls("/")) The idea here is that now every call to ``ls`` will have the "-la" arguments already specified. Baking can become very useful when you combine it with :ref:`subcommands`: .. code-block:: python from sh import ssh # calling whoami on a server. this is a lot to type out, especially if # you wanted to call many commands (not just whoami) back to back on # the same server iam1 = ssh("myserver.com", "-p 1393", "whoami") # wouldn't it be nice to bake the common parameters into the ssh command? myserver = ssh.bake("myserver.com", p=1393) print(myserver) # "/usr/bin/ssh myserver.com -p 1393" # resolves to "/usr/bin/ssh myserver.com -p 1393 whoami" iam2 = myserver.whoami() assert(iam1 == iam2) # True! Now that the "myserver" callable represents a baked ssh command, you can call anything on the server easily: .. code-block:: python # executes "/usr/bin/ssh myserver.com -p 1393 tail /var/log/dumb_daemon.log -n 100" print(myserver.tail("/var/log/dumb_daemon.log", n=100)) ================================================ FILE: docs/source/sections/command_class.rst ================================================ API ### .. _command_class: Command Class ============== The ``Command`` class represents a program that exists on the system and can be run at some point in time. An instance of ``Command`` is never running; an instance of :ref:`RunningCommand ` is spawned for that. An instance of ``Command`` can take the form of a manually instantiated object, or as an object instantiated by dynamic lookup: .. code-block:: python import sh ls1 = sh.Command("ls") ls2 = sh.ls assert ls1 == ls2 .. py:class:: Command(name, search_paths=None) Instantiates a Command instance, where *name* is the name of a program that exists on the user's ``$PATH``, or is a full path itself. If *search_paths* is specified, it must be a list of all the paths to look for the program name. .. code-block:: python from sh import Command ifconfig = Command("ifconfig") ifconfig = Command("/sbin/ifconfig") .. py:method:: Command.bake(*args, **kwargs) Returns a new Command with ``*args`` and ``**kwargs`` baked in as positional and keyword arguments, respectively. Any future calls to the returned Command will include ``*args`` and ``**kwargs`` automatically: .. code-block:: python from sh import ls long_ls = ls.bake("-l") print(ls("/var")) print(ls("/tmp")) .. seealso:: :ref:`baking` Similar to the above, arguments to the ``sh.Command`` must be separate. e.g. the following does not work:: lscmd = sh.Command("/bin/ls -l") tarcmd = sh.Command("/bin/tar cvf /tmp/test.tar /my/home/directory/") You will run into ``CommandNotFound(path)`` exception even when correct full path is specified. The correct way to do this is to : #. build ``Command`` object using *only* the binary #. pass the arguments to the object *when invoking* as follows:: lscmd = sh.Command("/bin/ls") lscmd("-l") tarcmd = sh.Command("/bin/tar") tarcmd("cvf", "/tmp/test.tar", "/my/home/directory/") .. _running_command: RunningCommand Class ==================== This represents a :ref:`Command ` instance that has been or is being executed. It exists as a wrapper around the low-level :ref:`OProc `. Most of your interaction with sh objects are with instances of this class. It is only returned if ``_return_cmd=True`` when you execute a command. .. warning:: Objects of this class behave very much like strings. This was an intentional design decision to make the "output" of an executing Command behave more intuitively. Be aware that functions that accept real strings only, for example ``json.dumps``, will not work on instances of RunningCommand, even though it look like a string. .. _wait_method: .. py:method:: RunningCommand.wait(timeout=None) :param timeout: An optional non-negative number to wait for the command to complete. If it doesn't complete by the timeout, we raise :ref:`timeout_exc`. Block and wait for the command to finish execution and obtain an exit code. If the exit code represents a failure, we raise the appropriate exception. See :ref:`exceptions `. .. note:: Calling this method multiple times only yields an exception on the first call. This is called automatically by sh unless your command is being executed :ref:`asynchronously `, in which case, you may want to call this manually to ensure completion. If an instance of :ref:`Command ` is being used as the stdin argument (see :ref:`piping `), :meth:`wait` is also called on that instance, and any exceptions resulting from that process are propagated up. .. py:attribute:: RunningCommand.process The underlying :ref:`OProc ` instance. .. py:attribute:: RunningCommand.stdout A ``@property`` that calls :meth:`wait` and then returns the contents of what the process wrote to stdout. .. py:attribute:: RunningCommand.stderr A ``@property`` that calls :meth:`wait` and then returns the contents of what the process wrote to stderr. .. py:attribute:: RunningCommand.exit_code A ``@property`` that calls :meth:`wait` and then returns the process's exit code. .. py:attribute:: RunningCommand.pid The process id of the process. .. py:attribute:: RunningCommand.sid The session id of the process. This will typically be a different session than the current python process, unless :ref:`_new_session=False ` was specified. .. py:attribute:: RunningCommand.pgid The process group id of the process. .. py:attribute:: RunningCommand.ctty The controlling terminal device, if there is one. .. py:method:: RunningCommand.signal(sig_num) Sends *sig_num* to the process. Typically used with a value from the :mod:`signal` module, like :py:data:`signal.SIGHUP` (see :manpage:`signal(7)`). .. py:method:: RunningCommand.signal_group(sig_num) Sends *sig_num* to every process in the process group. Typically used with a value from the :mod:`signal` module, like :py:data:`signal.SIGHUP` (see :manpage:`signal(7)`). .. py:method:: RunningCommand.terminate() Shortcut for :meth:`RunningCommand.signal(signal.SIGTERM) `. .. py:method:: RunningCommand.kill() Shortcut for :meth:`RunningCommand.signal(signal.SIGKILL) `. .. py:method:: RunningCommand.kill_group() Shortcut for :meth:`RunningCommand.signal_group(signal.SIGKILL) `. .. py:method:: RunningCommand.is_alive() Returns whether or not the process is still alive. :rtype: bool .. _oproc_class: OProc Class =========== .. warning:: Don't use instances of this class directly. It is being documented here for posterity, not for direct use. .. py:method:: OProc.wait() Block until the process completes, aggregate the output, and populate :attr:`OProc.exit_code`. .. py:attribute:: OProc.stdout A :class:`collections.deque`, sized to :ref:`_internal_bufsize ` items, that contains the process's STDOUT. .. py:attribute:: OProc.stderr A :class:`collections.deque`, sized to :ref:`_internal_bufsize ` items, that contains the process's STDERR. .. py:attribute:: OProc.exit_code Contains the process's exit code, or ``None`` if the process has not yet exited. .. py:attribute:: OProc.pid The process id of the process. .. py:attribute:: OProc.sid The session id of the process. This will typically be a different session than the current python process, unless :ref:`_new_session=False ` was specified. .. py:attribute:: OProc.pgid The process group id of the process. .. py:attribute:: OProc.ctty The controlling terminal device, if there is one. .. py:method:: OProc.signal(sig_num) Sends *sig_num* to the process. Typically used with a value from the :mod:`signal` module, like :py:data:`signal.SIGHUP` (see :manpage:`signal(7)`). .. py:method:: OProc.signal_group(sig_num) Sends *sig_num* to every process in the process group. Typically used with a value from the :mod:`signal` module, like :py:data:`signal.SIGHUP` (see :manpage:`signal(7)`). .. py:method:: OProc.terminate() Shortcut for :meth:`OProc.signal(signal.SIGTERM) `. .. py:method:: OProc.kill() Shortcut for :meth:`OProc.signal(signal.SIGKILL) `. .. py:method:: OProc.kill_group() Shortcut for :meth:`OProc.signal_group(signal.SIGKILL) `. Exceptions ========== .. _error_return_code: ErrorReturnCode --------------- .. py:class:: ErrorReturnCode This is the base class for, as the name suggests, error return codes. It subclasses :py:class:`Exception`. .. py:attribute:: ErrorReturnCode.full_cmd The full command that was executed, as a string, so that you can try it on the commandline if you wish. .. py:attribute:: ErrorReturnCode.stdout The total aggregated STDOUT for the process. .. py:attribute:: ErrorReturnCode.stderr The total aggregated STDERR for the process. .. py:attribute:: ErrorReturnCode.exit_code The process's adjusted exit code. .. seealso:: :ref:`arch_exit_code` .. _signal_exc: SignalException --------------- Subclasses :ref:`ErrorReturnCode `. Raised when a command receives a signal that causes it to exit. .. _timeout_exc: TimeoutException ---------------- Raised when a command specifies a non-null :ref:`timeout` and the command times out: .. code-block:: python import sh try: sh.sleep(10, _timeout=1) except sh.TimeoutException: print("we timed out, as expected") Also raised when you specify a timeout to :ref:`RunningCommand.wait(timeout=None)`: .. code-block:: python import sh p = sh.sleep(10, _bg=True) try: p.wait(timeout=1) except sh.TimeoutException: print("we timed out waiting") p.kill() .. _not_found_exc: CommandNotFound --------------- This exception is raised in one of the following conditions: * The program cannot be found on your path. * You do not have permissions to execute the program. * The program is not marked executable. The last two bullets may seem strange, but they fall in line with how a shell like Bash behaves when looking up a program to execute. .. note:: ``CommandNotFound`` subclasses ``AttributeError``. As such, the `repr` of it is simply the name of the missing attribute. Helper Functions ================ .. py:function:: which(name, search_paths=None) Resolves *name* to program's absolute path, or ``None`` if it cannot be found. If *search_paths* is list of paths, use that list to look for the program, otherwise use the environment variable ``$PATH``. .. py:function:: pushd(directory) This function provides a ``with`` context that behaves similar to Bash's `pushd `_ by pushing to the provided directory, and popping out of it at the end of the context. .. code-block:: python import sh with sh.pushd("/tmp"): sh.touch("a_file") .. note:: It should be noted that we use a reentrant lock, so that different threads using this function will have the correct behavior inside of their ``with`` contexts. ================================================ FILE: docs/source/sections/contrib.rst ================================================ .. _contrib: Contrib Commands ################ Contrib is an sh sub-module that provides friendly wrappers to useful commands. Typically, the commands being wrapped are unintuitive, and the contrib version makes them intuitive. .. note:: Contrib commands should be considered generally unstable. They will grow and change as the community figures out the best interface for them. Commands ======== Sudo ---- Allows you to enter your password from the terminal at runtime, or as a string in your script. .. py:function:: sudo(password=None, *args, **kwargs) Call sudo with ``password``, if specified, else ask the executing user for a password at runtime via :func:`getpass.getpass`. .. seealso:: :ref:`contrib_sudo` .. _contrib_git: Git --- Many git commands use a pager for output, which can cause an unexpected behavior when run through sh. To account for this, the contrib version sets ``_tty_out=False`` for all git commands. .. py:function:: git(*args, **kwargs) Call git with STDOUT connected to a pipe, instead of a TTY. .. code-block:: python from sh.contrib import git repo_log = git.log() .. seealso:: :ref:`faq_tty_out` and :ref:`faq_color_output` .. _contrib_ssh: SSH --- .. versionadded:: 1.13.0 SSH password-based logins :ref:`can be a pain `. This contrib command performs all of the ugly setup and provides a clean interface to using SSH. .. py:function:: ssh(interact=None, password=None, prompt_match=None, login_success=None, *args, **kwargs) :param interact: A callback to handle SSH session interaction *after* login is successful. Required. :param password: A password string or a function that returns a password string. Optional. If not provided, :func:`getpass.getpass` is used. :param prompt_match: The string to match in order to determine when to provide SSH with the password. Or a function that matches on the output. Optional. :param login_success: A function to determine if SSH login is successful. Optional. The ``interact`` parameter takes a callback with a signature that is slightly different to the function callbacks for :ref:`redirection `: .. py:function:: fn(content, stdin_queue) :param content: An instance of an ephemeral :ref:`SessionContent ` class whose job is to hold the characters that the SSH session has written to STDOUT. :param stdin_queue: A :class:`queue.Queue` object to communicate with STDIN programmatically. ``password`` can be simply a string that will be used to type the password. If it's not provided, it will be read from STDIN at runtime via :func:`getpass.getpass`. It can also be a callable that returns the password string. ``prompt_match`` is a string to match before the contrib command will provide the SSH process with the password. It is optional, and if left unspecified, will default to "password: ". It can also be a callable that is called on a :ref:`SessionContent ` instance and returns ``True`` or ``False`` for a match. ``login_success`` is a function that takes a :ref:`SessionContent ` object and returns a boolean for whether or not a successful login occurred. It is optional, and if unspecified, simply evaluates to ``True``, meaning any password submission results in a successful login (obviously not always correct). It is recommended that you specify this. .. _session_content: .. py:class:: SessionContent() This class contains a record lines and characters written to the SSH processes's STDOUT. It should be all you need from the callbacks to determine how to interact with the SSH process. .. py:attribute:: SessionContent.chars :type: :class:`collections.deque` The previous 50,000 characters. .. py:attribute:: SessionContent.lines :type: :class:`collections.deque` The previous 5,000 lines. .. py:attribute:: SessionContent.line_chars :type: list The characters in the line currently being aggregated. .. py:attribute:: SessionContent.cur_line :type: str A string of the line currently being aggregated. .. py:attribute:: SessionContent.last_line :type: str The previous line. .. py:attribute:: SessionContent.cur_char :type: str The currently written character. .. _contrib_bash: Bash --- Often users may find themselves having to run bash commands directly, whether due to commands having special characters (e.g. dash, or dot) or other reasons. This can lead into recurrently having to bake the ``bash`` command to call it directly. To account for this, the contrib version provides a ``bash`` command baked in: .. py:function:: bash(*args, **kwargs) Call bash with the prefix of "bash -c [...]". .. code-block:: python from sh.contrib import bash # Calling commands directly bash.ls() # equivallent to "bash -c ls" # Or adding the full commands bash("command-with-dashes args") Extending ========= For developers. To extend contrib, simply decorate a function in sh with the ``@contrib`` decorator, and pass in the name of the command you wish to shadow to the decorator. This method must return an instance of :ref:`Command `: .. code-block:: python @contrib("ls") def my_ls(original): ls = original.bake("-l") return ls Now you can run your custom contrib command from your scripts, and you'll be using the command returned from your decorated function: .. code-block:: python from sh.contrib import ls # executing: ls -l print(ls("/")) For even more flexibility, you can design your contrib command to rewrite its options based on *executed* arguments. For example, say you only wish to set a command's argument if another argument is set. You can accomplish it like this: .. code-block:: python @contrib("ls") def my_ls(original): def process(args, kwargs): if "-a" in args: args.append("-L") return args, kwargs ls = original.bake("-l") return ls, process Returning a process function along with the command will tell sh to use that function to preprocess the arguments at execution time using the :ref:`_arg_preprocess ` special kwarg. ================================================ FILE: docs/source/sections/default_arguments.rst ================================================ .. _default_arguments: Default Arguments ================= Many times, you want to override the default arguments of all commands launched through sh. For example, suppose you want the output of all commands to be aggregated into a :class:`io.StringIO` buffer. The naive way would be this: .. code-block:: python import sh from io import StringIO buf = StringIO() sh.ls("/", _out=buf) sh.whoami(_out=buf) sh.ps("auxwf", _out=buf) Clearly, this gets tedious quickly. Fortunately, we can create execution contexts that allow us to set default arguments on all commands spawned from that context: .. code-block:: python import sh from io import StringIO buf = StringIO() sh2 = sh.bake(_out=buf) sh2.ls("/") sh2.whoami() sh2.ps("auxwf") Now, anything launched from ``sh2`` will send its output to the ``StringIO`` instance ``buf``. ================================================ FILE: docs/source/sections/envs.rst ================================================ .. _environments: Environments ============ The :ref:`_env ` special kwarg allows you to pass a dictionary of environment variables and their corresponding values: .. code-block:: python import sh sh.google_chrome(_env={"SOCKS_SERVER": "localhost:1234"}) :ref:`_env ` replaces your process's environment completely. Only the key-value pairs in :ref:`_env ` will be used for its environment. If you want to add new environment variables for a process *in addition to* your existing environment, try something like this: .. code-block:: python import os import sh new_env = os.environ.copy() new_env["SOCKS_SERVER"] = "localhost:1234" sh.google_chrome(_env=new_env) .. seealso:: To make an environment apply to all sh commands look into :ref:`default_arguments`. ================================================ FILE: docs/source/sections/exit_codes.rst ================================================ .. _exit_codes: Exit Codes & Exceptions ======================= Normal processes exit with exit code 0. This can be seen from :attr:`RunningCommand.exit_code`: .. code-block:: python output = ls("/", _return_cmd=True) print(output.exit_code) # should be 0 If a process terminates, and the exit code is not 0, an exception is generated dynamically. This lets you catch a specific return code, or catch all error return codes through the base class :class:`ErrorReturnCode`: .. code-block:: python try: print(ls("/some/non-existant/folder")) except ErrorReturnCode_2: print("folder doesn't exist!") create_the_folder() except ErrorReturnCode: print("unknown error") You can also customize which exit codes indicate an error with :ref:`ok_code`. For example: .. code-block:: python for i in range(10): sh.grep("string to check", f"file_{i}.txt", _ok_code=(0, 1)) where the :ref:`ok_code` makes a failure to find a match a no-op. Signals ------- Signals are raised whenever your process terminates from a signal. The exception raised in this situation is :ref:`signal_exc`, which subclasses :ref:`error_return_code`. .. code-block:: python try: p = sh.sleep(3, _bg=True) p.kill() except sh.SignalException_SIGKILL: print("killed") This behavior could be blocked by appending the negative value of the signal to :ref:`ok_code`. All signals that raises :ref:`signal_exc` are ``[SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGINT, SIGKILL, SIGPIPE, SIGQUIT, SIGSEGV, SIGTERM, SIGTERM]``. .. note:: You can catch :ref:`signal_exc` by using either a number or a signal name. For example, the following two exception classes are equivalent: .. code-block:: python assert sh.SignalException_SIGKILL == sh.SignalException_9 ================================================ FILE: docs/source/sections/faq.rst ================================================ .. _faq: FAQ === How do I execute a bash builtin? -------------------------------- .. code-block:: python import sh sh.bash("-c", "your_builtin") Or .. code-block:: python import sh builtins = sh.bash.bake("-c") builtins("your_builtin") Will Windows be supported? -------------------------- There are no plans to support Windows. .. _faq_append: How do I append output to a file? --------------------------------- Use a file object opened in the mode you desire: .. code-block:: python import sh h = open("/tmp/output", "a") sh.ls("/dir1", _out=h) sh.ls("/dir2", _out=h) .. _faq_color_output: Why does my command's output have color? ---------------------------------------- Typically the reason for this is that your program detected that its STDOUT was connected to a TTY, and therefore decided to print color escape sequences in its output. The typical solution is to use :ref:`_tty_out=False `, which will force a pipe to be connected to STDOUT, and probably change the behavior of the program. .. seealso:: Git is one of the programs that makes extensive use of terminal colors (as well as pagers) in its output, so we added :ref:`a contrib version ` for convenience. .. _faq_tty_out: Why is _tty_out=True the default? --------------------------------- This was a design decision made for two reasons: 1. To make programs behave in the same way as seen on the commandline. 2. To provide better buffering control than pipes allow. For #1, we want sh to produce output that is identical to what the user sees from the commandline, because that's typically the only output they ever see from their command. This makes the output easy to understand. For #2, using a TTY for STDOUT allows us to precisely control the buffering of a command's output to sh's internal code. .. seealso:: :ref:`arch_buffers` Of course, there are some gotchas with TTY STDOUT. One of them is commands that use a pager, for example: .. code-block:: python import sh print(sh.git.log()) This will sometimes raise a ``SignalException_SIGPIPE``. The reason is because ``git log`` detects a TTY STDOUT and forks the system’s pager (typically ``less``) to handle the output. The pager checks for a controlling terminal, and, finding none, exits with exit code 1. The exit of the pager means no more readers on ``git log``’s output, and thus a ``SIGPIPE`` is received. One solution to the ``git log`` problem above is simply to use ``_tty_out=False``. Another option, specifically for git, is to use the ``git --no-pager`` option: .. code-block:: python import sh print(sh.git('--no-pager', 'log')) Why doesn't "*" work as a command argument? ------------------------------------------- Glob expansion is a feature of a shell, like Bash, and is performed by the shell before passing the results to the program to be exec'd. Because sh is not a shell, but rather tool to execute programs directly, we do not handle glob expansion like a shell would. So in order to use ``"*"`` like you would on the commandline, pass it into :func:`glob.glob` first: .. code-block:: python import sh import glob sh.ls(glob.glob("*.py")) .. _faq_path: How do I call a program that isn't in ``$PATH``? ------------------------------------------------ Use the :meth:`Command` constructor to instantiate an instance of Command directly, then execute that: .. code-block:: python import sh cmd = sh.Command("/path/to/command") cmd("-v", "arg1") How do I execute a program with a dash in its name? --------------------------------------------------- If it's in your ``$PATH``, substitute the dash for an underscore: .. code-block:: python import sh sh.google_chrome("http://google.com") The above will run ``google-chrome http://google.com`` .. note:: If a program named ``google_chrome`` exists on your system, that will be called instead. In that case, in order to execute the program with a dash in the name, you'll have to use the method described :ref:`here. ` .. _faq_special: How do I execute a program with a special character in its name? ---------------------------------------------------------------- Programs with non-alphanumeric, non-dash characters in their names cannot be executed directly as an attribute on the sh module. For example, **this will not work:** .. code-block:: python import sh sh.mkfs.ext4() The reason should be fairly obvious. In Python, characters like ``.`` have special meaning, in this case, attribute access. What sh is trying to do in the above example is find the program "mkfs" (which may or may not exist) and then perform a :ref:`subcommand lookup ` with the name "ext4". In other words, it will try to call ``mkfs`` with the argument ``ext4``, which is probably not what you want. The workaround is instantiating the :ref:`Command Class ` with the string of the program you're looking for: .. code-block:: python import sh mkfsext4 = sh.Command("mkfs.ext4") mkfsext4() # run it .. _faq_pipe_syntax: Why not use ``|`` to pipe commands? ----------------------------------- I prefer the syntax of sh to resemble function composition instead of a pipeline. One of the goals of sh is to make executing processes more like calling functions, not making function calls more like Bash. Why isn't piping asynchronous by default? ----------------------------------------- There is a non-obvious reason why async piping is not possible by default. Consider the following example: .. code-block:: python import sh sh.cat(sh.echo("test\n1\n2\n3\n")) When this is run, ``sh.echo`` executes and finishes, then the entire output string is fed into ``sh.cat``. What we would really like is each newline-delimited chunk to flow to ``sh.cat`` incrementally. But for this example to flow data asynchronously from echo to cat, the echo command would need to *not block.* But how can the inner command know the context of its execution, to know to block sometimes but not other times? It can't know that without something explicit. This is why the :ref:`piped` special kwarg was introduced. By default, commands executed block until they are finished, so in order for an inner command to not block, ``_piped=True`` signals to the inner command that it should not block. This way, the inner command starts running, then very shortly after, the outer command starts running, and both are running simultaneously. Data can then flow from the inner command to the outer command asynchronously: .. code-block:: python import sh sh.cat(sh.echo("test\n1\n2\n3\n", _piped=True)) Again, this example is contrived -- a better example would be a long-running command that produces a lot of output that you wish to pipe through another program incrementally. How do I run a command and connect it to sys.stdout and sys.stdin? ------------------------------------------------------------------ There are two ways to do this .. seealso:: :ref:`fg` You can use :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` as arguments to :ref:`in`, :ref:`out`, :ref:`err`, respectively, and it *should* mostly work as expected: .. code-block:: python import sh import sys sh.your_command(_in=sys.stdin, _out=sys.stdout) There are a few reasons why this probably won't work. The first reason is that :data:`sys.stdin` is probably a controlling TTY (attached to the shell that launched the python process), and probably not set in raw mode :manpage:`termios(3)`, which means that, among other things, input is buffered by newlines. The real solution is to use :ref:`_fg=True `: .. code-block:: python import sh sh.top(_fg=True) .. _faq_separate_args: Why do my arguments need to be separate strings? ------------------------------------------------ This confuses many new sh users. They want to do something like this and expect it to just work: .. code-block:: python from sh import tar tar("cvf /tmp/test.tar /my/home/directory") But instead they'll get a confusing error message: .. code-block:: none RAN: '/bin/tar cvf /tmp/test.tar /my/home/directory' STDOUT: STDERR: /bin/tar: Old option 'f' requires an argument. Try '/bin/tar --help' or '/bin/tar --usage' for more information. The reason why they expect it to work is because shells, like Bash, automatically parse your commandline and break up arguments for you, before sending them to the binary. They have a complex set of rules (some of which are represented by :mod:`shlex`) to take a single string of a command and arguments and separate them. Even if we wanted to implement this in sh (which we don't), it would hurt the ability for users to parameterize parts of their arguments. They would have to use string interpolation, which would be ugly and error prone: .. code-block:: python from sh import tar tar("cvf %s %s" % ("/tmp/tar1.tar", "/home/oh no a space") In the above example, ``"/home/oh"``, ``"no"``, ``"a"``, and ``"space"`` would all be separate arguments to tar, causing the program to behave unexpectedly. Basically every command with parameterized arguments would need to expect characters that could break the parser. .. _faq_arg_ordering: How do I order keyword arguments? --------------------------------- Typically this question gets asked when a user is trying to execute something like the following commandline: .. code-block:: none my-command --arg1=val1 arg2 --arg3=val3 This is usually the first attempt that they make: .. code-block:: python sh.my_command(arg1="val1", "arg2", arg3="val3") This doesn't work because, in Python, position arguments, like ``arg2`` cannot come after keyword arguments. Furthermore, it is entirely possible that ``--arg3=val3`` comes before ``--arg1=val1``. The reason for this is that a function's ``**kwargs`` is an unordered mapping, and so key-value pairs are not guaranteed to resolve to a specific order. So the solution here is to forego the usage of the keyword argument *convenience*, and just use raw ordered arguments: .. code-block:: python sh.my_command("--arg1=val1", "arg2", "--arg3=val3") .. _faq_pylint: How to disable pylint E1101 no-member errors? --------------------------------------------- Pylint complains with E1101 no-member to almost all ``sh.command`` invocations, because it doesn't know, that these members are generated dynamically. Starting with Pylint 1.6 these messages can be suppressed using `generated-members `_ option. Just add following lines to ``pylintrc``:: [TYPECHECK] generated-members=sh How do I patch sh in my tests? ------------------------------ sh can be patched in your tests the typical way, with :func:`unittest.mock.patch`: .. code-block:: python from unittest.mock import patch import sh def get_something(): return sh.pwd() @patch("sh.pwd", create=True) def test_something(pwd): pwd.return_value = "/" assert get_something() == "/" The important thing to note here is that ``create=True`` is set. This is required because sh is a bit magical and ``patch`` will fail to find the ``pwd`` command as an attribute on the sh module. You may also patch the :class:`Command` class: .. code-block:: python from unittest.mock import patch import sh def get_something(): pwd = sh.Command("pwd") return pwd() @patch("sh.Command") def test_something(Command): Command().return_value = "/" assert get_something() == "/" Notice here we do not need ``create=True``, because :class:`Command` is not an automatically generated object on the sh module (it actually exists). Why is sh just a single file? ----------------------------- When sh was first written, the design decision was made to make it a single-file module. This has pros and cons: Cons: - Auditing the code is more challenging - Without file-enforced structure, adding more features and abstractions makes the code harder to follow - Cognitively, it feels cluttered Pros: - Can be used easily on systems without Python package managers - Can be embedded/bundled together with other software more easily - Cognitively, it feels more self-contained In my mind, because the primary target audience of sh users is generally more scrappy devops, systems people, or people just trying to stitch together some clunky system programs, the listed pros weigh a little more heavily than the cons. Sacrificing some development advantages to give those users a more flexible tool is a win to me. Down the road, the development disadvantages of a single file can be solved with additional development tools, for example, with a tool that compiles multiple modules into the single sh.py file. Realistically, though, sh is pretty mature, so I don't see it growing much more in complexity or code size. How do I see the commands sh is running? ---------------------------------------- Use logging: .. code-block:: python import logging import sh logging.basicConfig(level=logging.INFO) sh.ls() .. code-block:: none INFO:sh.command:: starting process INFO:sh.command:: process started INFO:sh.command:: process completed ... ================================================ FILE: docs/source/sections/passing_arguments.rst ================================================ .. _passing_arguments: Passing Arguments ================= When passing multiple arguments to a command, each argument *must* be a separate string: .. code-block:: python from sh import tar tar("cvf", "/tmp/test.tar", "/my/home/directory/") This *will not work*: .. code-block:: python from sh import tar tar("cvf /tmp/test.tar /my/home/directory") .. seealso:: :ref:`faq_separate_args` Keyword Arguments ----------------- sh supports short-form ``-a`` and long-form ``--arg`` arguments as keyword arguments: .. code-block:: python # resolves to "curl http://duckduckgo.com/ -o page.html --silent" curl("http://duckduckgo.com/", o="page.html", silent=True) # or if you prefer not to use keyword arguments, this does the same thing: curl("http://duckduckgo.com/", "-o", "page.html", "--silent") # resolves to "adduser amoffat --system --shell=/bin/bash --no-create-home" adduser("amoffat", system=True, shell="/bin/bash", no_create_home=True) # or adduser("amoffat", "--system", "--shell", "/bin/bash", "--no-create-home") .. seealso:: :ref:`faq_arg_ordering` ================================================ FILE: docs/source/sections/piping.rst ================================================ .. _piping: Piping ====== Basic ----- Bash style piping is performed using function composition. Just pass one command as the input to another's ``_in`` argument, and sh will send the output of the inner command to the input of the outer command: .. code-block:: python # sort this directory by biggest file print(sort("-rn", _in=du(glob("*"), "-sb"))) # print(the number of folders and files in /etc print(wc("-l", _in=ls("/etc", "-1"))) .. note:: This basic piping does not flow data through asynchronously; the inner command blocks until it finishes, before sending its data to the outer command. By default, any command that is piping another command in waits for it to complete. This behavior can be changed with the :ref:`_piped ` special kwarg on the command being piped, which tells it not to complete before sending its data, but to send its data incrementally. Read ahead for examples of this. .. _advanced_piping: Advanced -------- By default, all piped commands execute sequentially. What this means is that the inner command executes first, then sends its data to the outer command: .. code-block:: python print(wc("-l", _in=ls("/etc", "-1"))) In the above example, ``ls`` executes, gathers its output, then sends that output to ``wc``. This is fine for simple commands, but for commands where you need parallelism, this isn't good enough. Take the following example: .. code-block:: python for line in tr(_in=tail("-f", "test.log"), "[:upper:]", "[:lower:]", _iter=True): print(line) **This won't work** because the ``tail -f`` command never finishes. What you need is for ``tail`` to send its output to ``tr`` as it receives it. This is where the :ref:`_piped ` special kwarg comes in handy: .. code-block:: python for line in tr(_in=tail("-f", "test.log", _piped=True), "[:upper:]", "[:lower:]", _iter=True): print(line) This works by telling ``tail -f`` that it is being used in a pipeline, and that it should send its output line-by-line to ``tr``. By default, :ref:`piped` sends STDOUT, but you can easily make it send STDERR instead by using ``_piped="err"`` ================================================ FILE: docs/source/sections/redirection.rst ================================================ .. _redirection: Redirection =========== sh can redirect the STDOUT and STDERR of a process to many different types of targets, using the :ref:`_out ` and :ref:`_err ` special kwargs. Filename -------- If a string is used, it is assumed to be a filename. The filename is opened as "wb", meaning truncate-write and binary mode. .. code-block:: python import sh sh.ifconfig(_out="/tmp/interfaces") .. seealso:: :ref:`faq_append` File-like Object ---------------- You may also use any object that supports ``.write(data)``, like :class:`io.StringIO`: .. code-block:: python import sh from io import StringIO buf = StringIO() sh.ifconfig(_out=buf) print(buf.getvalue()) .. _red_func: Function Callback ----------------- A callback function may also be used as a target. The function must conform to one of three signatures: .. py:function:: fn(data) :noindex: The function takes just the chunk of data from the process. .. py:function:: fn(data, stdin_queue) :noindex: In addition to the previous signature, the function also takes a :class:`queue.Queue`, which may be used to communicate programmatically with the process. .. py:function:: fn(data, stdin_queue, process) :noindex: In addition to the previous signature, the function takes a :class:`weakref.ref` to the :ref:`OProc ` object. .. seealso:: :ref:`callbacks` .. seealso:: :ref:`tutorial2` ================================================ FILE: docs/source/sections/special_arguments.rst ================================================ .. _special_arguments: .. |def| replace:: Default value: Special Kwargs ############## These arguments alter a command's behavior. They are not passed to the program. You can use them on any command that you run, but some may not be used together. sh will tell you if there are conflicts. To set default special keyword arguments on *every* command run, you may use :ref:`default_arguments`. Controlling Output ================== .. _out: _out ---- |def| ``None`` What to redirect STDOUT to. If this is a string, it will be treated as a file name. You may also pass a file object (or file-like object), an int (representing a file descriptor, like the result of :func:`os.pipe`), a :class:`io.StringIO` object, or a callable. .. code-block:: python import sh sh.ls(_out="/tmp/output") .. seealso:: :ref:`redirection` .. _err: _err ---- |def| ``None`` What to redirect STDERR to. See :ref:`_out`. _err_to_out ----------- |def| ``False`` If ``True``, duplicate the file descriptor bound to the process's STDOUT also to STDERR, effectively causing STDERR and STDOUT to go to the same place. _encoding --------- |def| ``sh.DEFAULT_ENCODING`` The character encoding of the process's STDOUT. By default, this is the locale's default encoding. _decode_errors -------------- .. versionadded:: 1.07.0 |def| ``"strict"`` This is how Python should handle decoding errors of the process's output. By default, this is ``"strict"``, but you can use any value that's valid to :meth:`bytes.decode`, such as ``"ignore"``. _tee ---- .. versionadded:: 1.07.0 |def| ``None`` As of 1.07.0, any time redirection is used, either for STDOUT or STDERR, the respective internal buffers are not filled. For example, if you're downloading a file and using a callback on STDOUT, the internal STDOUT buffer, nor the pipe buffer be filled with data from STDOUT. This option forces one of stderr (``_tee='err'``) or stdout (``_tee='out'`` or ``_tee=True``) to be filled anyways, in effect "tee-ing" the output into two places (the callback/redirect handler, and the internal buffers). _truncate_exc ------------- .. versionadded:: 1.12.0 |def| ``True`` Whether or not exception ouput should be truncated. Execution ========= .. _fg: _fg --- .. versionadded:: 1.12.0 |def| ``False`` Runs a command in the foreground, meaning it is spawned using :func:`os.spawnle()`. The current process's STDIN/OUT/ERR is :func:`os.dup2`'d to the new process and so the new process becomes the *foreground* of the shell executing the script. This is only really useful when you want to launch a lean, interactive process that sh is having trouble running, for example, ssh. .. warning:: ``_fg=True`` side-steps a lot of sh's functionality. You will not be returned a process object and most (likely all) other special kwargs will not work. If you are looking for similar functionality, but still retaining sh's features, use the following: .. code-block:: python import sh import sys sh.your_command(_in=sys.stdin, _out=sys.stdout, _err=sys.stderr) .. _bg: _bg --- |def| ``False`` Runs a command in the background. The command will return immediately, and you will have to run :meth:`RunningCommand.wait` on it to ensure it terminates. .. seealso:: :ref:`background`. .. _bg_exc: _bg_exc ------- .. versionadded:: 1.12.9 |def| ``True`` Automatically report exceptions for the background command. If you set this to ``False`` you should make sure to call :meth:`RunningCommand.wait` or you may swallow exceptions that happen in the background command. .. _async_kw: _async ------ .. versionadded:: 2.0.0 |def| ``False`` Allows your command to become awaitable. Use in combination with :ref:`_iter ` and ``async for`` to incrementally await output as it is produced. .. _env: _env ---- |def| ``None`` A dictionary defining the only environment variables that will be made accessible to the process. If not specified, the calling process's environment variables are used. .. note:: This dictionary is the authoritative environment for the process. If you wish to change a single variable in your current environement, you must pass a copy of your current environment with the overriden variable to sh. .. seealso:: :ref:`environments` .. _timeout: _timeout -------- |def| ``None`` How much time, in seconds, we should give the process to complete. If the process does not finish within the timeout, it will be sent the signal defined by :ref:`timeout_signal`. .. _timeout_signal: _timeout_signal --------------- |def| ``signal.SIGKILL`` The signal to be sent to the process if :ref:`timeout` is not ``None``. _cwd ---- |def| ``None`` A string that sets the current working directory of the process. .. _ok_code: _ok_code -------- |def| ``0`` Either an integer, a list, or a tuple containing the exit code(s) that are considered "ok", or in other words: do not raise an exception. Some misbehaved programs use exit codes other than 0 to indicate success. .. code-block:: python import sh sh.weird_program(_ok_code=[0,3,5]) If the process is killed by a signal, a :ref:`signal_exc` is raised by default. This behavior could be blocked by appending a negative number to :ref:`ok_code` that represents the signal. .. code-block:: python import sh # the process won't raise SignalException if SIGINT, SIGKILL, or SIGTERM # are sent to kill the process p = sh.sleep(3, _bg=True, _ok_code=[0, -2, -9, -15]) # No exception will be raised here p.kill() .. seealso:: :ref:`exit_codes` .. _new_session: _new_session ------------ |def| ``False`` Determines if our forked process will be executed in its own session via :func:`os.setsid`. .. versionchanged:: 2.0.0 The default value of ``_new_session`` was changed from ``True`` to ``False`` because it makes more sense for a launched process to default to being in the process group of python script, so that it receives SIGINTs correctly. .. note:: If ``_new_session`` is ``False``, the forked process will be put into its own group via ``os.setpgrp()``. This way, the forked process, and all of it's children, are always alone in their own group that may be signalled directly, regardless of the value of ``_new_session``. .. seealso:: :ref:`architecture` .. _uid: _uid ---- .. versionadded:: 1.12.0 |def| ``None`` The user id to assume before the child process calls :func:`os.execv`. _preexec_fn ----------- .. versionadded:: 1.12.0 |def| ``None`` A function to be run directly before the child process calls :func:`os.execv`. Typically not used by normal users. .. _pass_fds: _pass_fds --------- .. versionadded:: 1.13.0 |def| ``{}`` (empty set) A whitelist iterable of integer file descriptors to be inherited by the child. Passing anything in this argument causes :ref:`_close_fds ` to be ``True``. .. _close_fds: _close_fds ---------- .. versionadded:: 1.13.0 |def| ``True`` Causes all inherited file descriptors besides stdin, stdout, and stderr to be automatically closed. This option is automatically enabled when :ref:`_pass_fds ` is given a value. Communication ============= .. _in: _in --- |def| ``None`` Specifies an argument for the process to use as its standard input. This may be a string, a :class:`queue.Queue`, a file-like object, or any iterable. .. seealso:: :ref:`stdin` .. _piped: _piped ------ |def| ``None`` May be ``True``, ``"out"``, or ``"err"``. Signals a command that it is being used as the input to another command, so it should return its output incrementally as it receives it, instead of aggregating it all at once. .. seealso:: :ref:`Advanced Piping ` .. _iter: _iter ----- |def| ``None`` May be ``True``, ``"out"``, or ``"err"``. Puts a command in iterable mode. In this mode, you can use a ``for`` or ``while`` loop to iterate over a command's output in real-time. .. code-block:: python import sh for line in sh.cat("/tmp/file", _iter=True): print(line) .. seealso:: :ref:`iterable`. .. _iter_noblock: _iter_noblock ------------- |def| ``None`` Same as :ref:`_iter `, except the loop will not block if there is no output to iterate over. Instead, the output from the command will be :py:data:`errno.EWOULDBLOCK`. .. code-block:: python import sh import errno import time for line in sh.tail("-f", "stuff.log", _iter_noblock=True): if line == errno.EWOULDBLOCK: print("doing something else...") time.sleep(0.5) else: print("processing line!") .. seealso:: :ref:`iterable`. .. _with: _with ----- |def| ``False`` Explicitly tells us that we're running a command in a ``with`` context. This is only necessary if you're using a command in a ``with`` context **and** passing parameters to it. .. code-block:: python import sh with sh.contrib.sudo(password="abc123", _with=True): print(sh.ls("/root")) .. seealso:: :ref:`with_contexts` .. _done: _done ----- .. versionadded:: 1.11.0 |def| ``None`` A callback that is *always* called when the command completes, even if it completes with an exit code that would raise an exception. After the callback is run, any exception that would be raised is raised. The callback is passed the :ref:`RunningCommand ` instance, a boolean indicating success, and the exit code. .. include:: /examples/done.rst TTYs ==== .. _tty_in: _tty_in ------- |def| ``False``, meaning a :func:`os.pipe` will be used. If ``True``, sh creates a TTY for STDIN, essentially emulating a terminal, as if your command was entered from the commandline. This is necessary for commands that require STDIN to be a TTY. .. _tty_out: _tty_out -------- |def| ``True`` If ``True``, sh creates a TTY for STDOUT, otherwise use a :func:`os.pipe`. This is necessary for commands that require STDOUT to be a TTY. .. seealso:: :ref:`faq_tty_out` .. _unify_ttys: _unify_ttys ----------- .. versionadded:: 1.13.0 |def| ``False`` If ``True``, sh will combine the STDOUT and STDIN TTY into a single pseudo-terminal. This is sometimes required by picky programs which expect to be dealing with a single pseudo-terminal, like SSH. .. seealso:: :ref:`tutorial2` _tty_size --------- |def| ``(20, 80)`` The (rows, columns) of stdout's TTY. Changing this may affect how much your program prints per line, for example. Performance & Optimization ========================== _in_bufsize ----------- |def| ``0`` The STDIN buffer size. 0 for unbuffered, 1 for line buffered, anything else for a buffer of that amount. .. _out_bufsize: _out_bufsize ------------ |def| ``1`` The STDOUT buffer size. 0 for unbuffered, 1 for line buffered, anything else for a buffer of that amount. .. _err_bufsize: _err_bufsize ------------ |def| ``1`` Same as :ref:`out_bufsize`, but with STDERR. .. _internal_bufsize: _internal_bufsize ----------------- |def| ``3 * 1024**2`` chunks How much of STDOUT/ERR your command will store internally. This value represents the *number of bufsize chunks* not the total number of bytes. For example, if this value is 100, and STDOUT is line buffered, you will be able to retrieve 100 lines from STDOUT. If STDOUT is unbuffered, you will be able to retrieve only 100 characters. _no_out ------- .. versionadded:: 1.07.0 |def| ``False`` Disables STDOUT being internally stored. This is useful for commands that produce huge amounts of output that you don't need, that would otherwise be hogging memory if stored internally by sh. _no_err ------- .. versionadded:: 1.07.0 |def| ``False`` Disables STDERR being internally stored. This is useful for commands that produce huge amounts of output that you don't need, that would otherwise be hogging memory if stored internally by sh. _no_pipe -------- .. versionadded:: 1.07.0 |def| ``False`` Similar to ``_no_out``, this explicitly tells the sh command that it will never be used for piping its output into another command, so it should not fill its internal pipe buffer with the process's output. This is also useful for conserving memory. Program Arguments ================= These are options that affect how command options are fed into the program. _long_sep --------- .. versionadded:: 1.12.0 |def| ``"="`` This is the character(s) that separate a program's long argument's key from the value, when using kwargs to specify your program's long arguments. For example, if your program expects a long argument in the form ``--name value``, the way to achieve this would be to set ``_long_sep=" "``. .. code-block:: python import sh sh.your_program(key=value, _long_sep=" ") Would send the following list of arguments to your program: .. code-block:: python ["--key value"] If your program expects the long argument name to be separate from its value, pass ``None`` into ``_long_sep`` instead: .. code-block:: python import sh sh.your_program(key=value, _long_sep=None) Would send the following list of arguments to your program: .. code-block:: python ["--key", "value"] _long_prefix ------------ .. versionadded:: 1.12.0 |def| ``"--"`` This is the character(s) that prefix a long argument for the program being run. Some programs use single dashes, for example, and do not understand double dashes. .. _preprocess: _arg_preprocess --------------- .. versionadded:: 1.12.0 |def| ``None`` This is an advanced option that allows you to rewrite a command's arguments on the fly, based on other command arguments, or some other variable. It is really only useful in conjunction with :ref:`baking `, and only currently used when constructing :ref:`contrib ` wrappers. Example: .. code-block:: python import sh def processor(args, kwargs): return args, kwargs my_ls = sh.bake.ls(_arg_preprocess=processor) .. warning:: The interface to the ``_arg_preprocess`` function may change without warning. It is generally only for internal sh use, so don't use it unless you absolutely have to. Misc ==== _log_msg -------- |def| ``None`` .. versionadded:: 1.12.0 This allows for a custom logging header for :ref:`command_class` instances. For example, the default logging looks like this: .. code-block:: python import logging import sh logging.basicConfig(level=logging.INFO) sh.ls("-l") .. code-block:: none INFO:sh.command:: starting process INFO:sh.command:: process started INFO:sh.command:: process completed People can find this ``` to make it well-behaved. In particular, the contrib version allows you to specify your password at execution time via terminal input, or as a string in your script. Terminal Input ^^^^^^^^^^^^^^ Via a :ref:`with context `: .. code-block:: python import sh with sh.contrib.sudo: print(ls("/root")) Or alternatively via :ref:`subcommands `: .. code-block:: python import sh print(sh.contrib.sudo.ls("/root")) Output: .. code-block:: none [sudo] password for youruser: ************* your_root_files.txt In the above example, ``sh.contrib.sudo`` automatically asks you for a password using :func:`getpass.getpass` under the hood. This method is the most secure, because it lowers the chances of doing something insecure, like including your password in your python script, or by saying that a particular user can execute anything inside of a particular script (the NOPASSWD method). .. note:: ``sh.contrib.sudo`` does not do password caching like the sudo binary does. Thie means that each time a sudo command is run in your script, you will be asked to type in a password. String Input ^^^^^^^^^^^^ You may also specify your password to ``sh.contrib.sudo`` as a string: .. code-block:: python import sh password = get_your_password() with sh.contrib.sudo(password=password, _with=True): print(ls("/root")) .. warning:: This method is less secure because it becomes tempting to hard-code your password into the python script, and that's a bad idea. However, it is more flexible, because it allows you to obtain your password from another source, so long as the end result is a string. /etc/sudoers NOPASSWD --------------------- With this method, you can use the raw ``sh.sudo`` command directly, because you're being guaranteed that the system will not ask you for a password. It first requires you set up your user to have root execution privileges Edit your sudoers file: .. code-block:: none $> sudo visudo Add or edit the line describing your user's permissions: .. code-block:: none yourusername ALL = (root) NOPASSWD: /path/to/your/program This says ``yourusername`` on ``ALL`` hosts will be able to run as root, but only root ``(root)`` (no other users), and that no password ``NOPASSWD`` will be asked of ``/path/to/your/program``. .. warning:: This method can be insecure if an unprivileged user can edit your script, because the entire script will be exited as a privileged user. A malicious user could put something bad in this script. .. _sudo_raw: sh.sudo ------- Using the raw command ``sh.sudo`` (which resolves directly to the system's ``sudo`` binary) without NOPASSWD is possible, provided you wire up the special keyword arguments on your own to make it behave correctly. This method is discussed generally for educational purposes; if you take the time to wire up ``sh.sudo`` on your own, then you have in essence just recreated :ref:`contrib_sudo`. .. code-block:: python import sh # password must end in a newline my_password = "password\n" # -S says "get the password from stdin" my_sudo = sh.sudo.bake("-S", _in=my_password) print(my_sudo.ls("root")) _fg=True -------- Another less-obvious way of using sudo is by executing the raw ``sh.sudo`` command but also putting it in the foreground. This way, sudo will work correctly automatically, by hooking up stdin/out/err automatically, and by asking you for a password if it requires one. The downsides of using :ref:`_fg=True `, however, are that you cannot capture its output -- everything is just printed to your terminal as if you ran it from a shell. .. code-block:: python import sh sh.sudo.ls("/root", _fg=True) ================================================ FILE: docs/source/sections/with.rst ================================================ .. _with_contexts: 'With' Contexts =============== Commands can be run within a Python ``with`` context. Popular commands using this might be ``sudo`` or ``fakeroot``: .. code-block:: python with sh.contrib.sudo: print(ls("/root")) .. seealso:: :ref:`contrib_sudo` If you need to run a command in a with context and pass in arguments, for example, specifying a -p prompt with sudo, you need to use the :ref:`_with=True ` This let's the command know that it's being run from a with context so it can behave correctly: .. code-block:: python with sh.contrib.sudo(k=True, _with=True): print(ls("/root")) ================================================ FILE: docs/source/tutorials/interacting_with_processes.rst ================================================ .. _tutorial2: Entering an SSH password ======================== Here we will attempt to SSH into a server and enter a password programmatically. .. note:: It is recommended that you just ``ssh-copy-id`` to copy your public key to the server so you don't need to enter your password, but for the purposes of this demonstration, we try to enter a password. To interact with a process, we need to assign a callback to STDOUT. The callback signature we'll use will take a :class:`queue.Queue` object for the second argument, and we'll use that to send STDIN back to the process. .. seealso:: :ref:`red_func` Here's our first attempt: .. code-block:: python from sh import ssh def ssh_interact(line, stdin): line = line.strip() print(line) if line.endswith("password:"): stdin.put("correcthorsebatterystaple") ssh("10.10.10.100", _out=ssh_interact) If you run this (substituting an IP that you can SSH to), you'll notice that nothing is printed from within the callback. The problem has to do with STDOUT buffering. By default, sh line-buffers STDOUT, which means that ``ssh_interact`` will only receive output when sh encounters a newline in the output. This is a problem because the password prompt has no newline: .. code-block:: none amoffat@10.10.10.100's password: Because a newline is never encountered, nothing is sent to the ``ssh_interact`` callback. So we need to change the STDOUT buffering. We do this with the :ref:`_out_bufsize ` special kwarg. We'll set it to 0 for unbuffered output: .. code-block:: python from sh import ssh def ssh_interact(line, stdin): line = line.strip() print(line) if line.endswith("password:"): stdin.put("correcthorsebatterystaple") ssh("10.10.10.100", _out=ssh_interact, _out_bufsize=0) If you run this updated version, you'll notice a new problem. The output looks like this: .. code-block:: none a m o f f a t @ 1 0 . 1 0 . 1 0 . 1 0 0 ' s p a s s w o r d : This is because the chunks of STDOUT our callback is receiving are unbuffered, and are therefore individual characters, instead of entire lines. What we need to do now is aggregate this character-by-character data into something more meaningful for us to test if the pattern ``password:`` has been sent, signifying that SSH is ready for input. It would make sense to encapsulate the variable we'll use for aggregating into some kind of closure or class, but to keep it simple, we'll just use a global: .. code-block:: python from sh import ssh import sys aggregated = "" def ssh_interact(char, stdin): global aggregated sys.stdout.write(char.encode()) sys.stdout.flush() aggregated += char if aggregated.endswith("password: "): stdin.put("correcthorsebatterystaple") ssh("10.10.10.100", _out=ssh_interact, _out_bufsize=0) You'll also notice that the example still doesn't work. There are two problems: The first is that your password must end with a newline, as if you had typed it and hit the return key. This is because SSH has no idea how long your password is, and is line-buffering STDIN. The second problem lies deeper in SSH. SSH needs a TTY attached to its STDIN in order to work properly. This tricks SSH into believing that it is interacting with a real user in a real terminal session. To enable TTY, we can add the :ref:`_tty_in ` special kwarg. We also need to use :ref:`_unify_ttys ` special kwarg. This tells sh to make STDOUT and STDIN come from a single pseudo-terminal, which is a requirement of SSH: .. code-block:: python from sh import ssh import sys aggregated = "" def ssh_interact(char, stdin): global aggregated sys.stdout.write(char.encode()) sys.stdout.flush() aggregated += char if aggregated.endswith("password: "): stdin.put("correcthorsebatterystaple\n") ssh("10.10.10.100", _out=ssh_interact, _out_bufsize=0, _tty_in=True, _unify_ttys=True) And now our remote login script works! .. code-block:: none amoffat@10.10.10.100's password: Linux 10.10.10.100 testhost #1 SMP Tue Jun 21 10:29:24 EDT 2011 i686 GNU/Linux Ubuntu 10.04.2 LTS Welcome to Ubuntu! * Documentation: https://help.ubuntu.com/ 66 packages can be updated. 53 updates are security updates. Ubuntu 10.04.2 LTS Welcome to Ubuntu! * Documentation: https://help.ubuntu.com/ You have new mail. Last login: Thu Sep 13 03:53:00 2012 from some.ip.address amoffat@10.10.10.100:~$ SSH Contrib command ------------------- The above process can be simplified by using a :ref:`contrib`. The :ref:`SSH contrib command ` does all the ugly kwarg argument setup for you, and provides a simple but powerful interface for doing SSH password logins. Please see the :ref:`SSH contrib command ` for more details about the exact api: .. code-block:: python from sh.contrib import ssh def ssh_interact(content, stdin): sys.stdout.write(content.cur_char) sys.stdout.flush() # automatically logs in with password and then presents subsequent content to # the ssh_interact callback ssh("10.10.10.100", password="correcthorsebatterystaple", interact=ssh_interact) How you should REALLY be using SSH ---------------------------------- Many people want to learn how to enter an SSH password by script because they want to execute remote commands on a server. Instead of trying to log in through SSH and then sending terminal input of the command to run, let's see how we can do it another way. First, open a terminal and run ``ssh-copy-id yourservername``. You'll be asked to enter your password for the server. After entering your password, you'll be able to SSH into the server without needing a password again. This simplifies things greatly for sh. The second thing we want to do is use SSH's ability to pass a command to run to the server you're SSHing to. Here's how you can run ``ifconfig`` on a server without having to use that server's shell directly: .. code-block:: none ssh amoffat@10.10.10.100 ifconfig Translating this to sh, it becomes: .. code-block:: python import sh print(sh.ssh("amoffat@10.10.10.100", "ifconfig")) We can make this even nicer by taking advantage of sh's :ref:`baking` to bind our server username/ip to a command object: .. code-block:: python import sh my_server = sh.ssh.bake("amoffat@10.10.10.100") print(my_server("ifconfig")) print(my_server("whoami")) Now we have a reusable command object that we can use to call remote commands. But there is room for one more improvement. We can also use sh's :ref:`subcommands` feature which expands attribute access into command arguments: .. code-block:: python import sh my_server = sh.ssh.bake("amoffat@10.10.10.100") print(my_server.ifconfig()) print(my_server.whoami()) ================================================ FILE: docs/source/tutorials/real_time_output.rst ================================================ .. _tutorial1: Tailing a real-time log file ============================ sh has the ability to respond to subprocesses in an event-driven fashion. A typical example of where this would be useful is tailing a log file for a specific pattern, then responding to that value immediately:: from sh import tail for line in tail("-f", "info.log", _iter=True): if "ERROR" in line: send_an_email_to_support(line) The :ref:`_iter ` special kwarg takes a command that would normally block until completion, and turns its output into a real-time iterable. .. seealso:: :ref:`iterable` Of course, you can do more than just tail log files. Any program that produces output can be iterated over. Say you wanted to send an email to a coworker if their C code emits a warning: .. code-block:: python from sh import gcc, git for line in gcc("-o", "awesome_binary", "awesome_source.c", _iter=True): if "warning" in line: # parse out the relevant info filename, line, char, message = line.split(":", 3) # find the commit using git commit = git("blame", "-e", filename, L="%d,%d" % (line,line)) # send them an email email_address = parse_email_from_commit_line(commit) send_email(email_address, message) Using :ref:`_iter ` is a great way to respond to events from another program, but your blocks while you're looping, making you unable to do anything else. To be truly event-driven, sh provides callbacks: .. code-block:: python from sh import tail def process_log_line(line): if "ERROR" in line: send_an_email_to_support(line) process = tail("-f", "info.log", _out=process_log_line, _bg=True) # ... do other stuff here ... process.wait() The :ref:`_out ` special kwarg lets you to assign a callback to STDOUT. This callback will receive each line of output from ``tail -f`` and allow you to do the same processing that we did earlier. .. seealso:: :ref:`callbacks` .. seealso:: :ref:`redirection` ================================================ FILE: docs/source/tutorials.rst ================================================ Tutorials ========= .. toctree:: tutorials/real_time_output tutorials/interacting_with_processes ================================================ FILE: docs/source/usage.rst ================================================ Usage ===== .. toctree:: sections/passing_arguments sections/exit_codes sections/redirection sections/asynchronous_execution sections/baking sections/piping sections/subcommands sections/default_arguments sections/envs sections/stdin sections/with ================================================ FILE: pyproject.toml ================================================ [tool.poetry] name = "sh" version = "2.2.2" description = "Python subprocess replacement" authors = ["Andrew Moffat "] readme = "README.rst" maintainers = [ "Andrew Moffat ", "Erik Cederstrand ", ] homepage = "https://sh.readthedocs.io/" repository = "https://github.com/amoffat/sh" documentation = "https://sh.readthedocs.io/" license = "MIT" classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Build Tools", "Topic :: Software Development :: Libraries :: Python Modules", ] include = [ { path = "CHANGELOG.md", format = "sdist" }, { path = "MIGRATION.md", format = "sdist" }, { path = "images", format = "sdist" }, { path = "Makefile", format = "sdist" }, { path = "tests", format = "sdist" }, { path = "tox.ini", format = "sdist" }, { path = "LICENSE.txt", format = "sdist" }, ] [tool.poetry.dependencies] python = ">=3.8.1,<4.0" [tool.poetry.group.dev.dependencies] tox = "^4.6.4" black = "^23.7.0" coverage = "^7.2.7" flake8 = "^6.1.0" rstcheck = "^6.1.2" sphinx = ">=1.6,<7" sphinx-rtd-theme = "^1.2.2" pytest = "^7.4.0" mypy = "^1.4.1" toml = "^0.10.2" [build-system] requires = ["poetry-core>=1.0.0a5"] build-backend = "poetry.core.masonry.api" ================================================ FILE: sh.py ================================================ """ https://sh.readthedocs.io/en/latest/ https://github.com/amoffat/sh """ # =============================================================================== # Copyright (C) 2011-2025 by Andrew Moffat # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # =============================================================================== import asyncio import platform from collections import deque from collections.abc import Mapping from importlib import metadata try: __version__ = metadata.version("sh") except metadata.PackageNotFoundError: # pragma: no cover __version__ = "unknown" if "windows" in platform.system().lower(): # pragma: no cover raise ImportError( f"sh {__version__} is currently only supported on Linux and macOS." ) import errno import fcntl import gc import getpass import glob as glob_module import inspect import logging import os import pty import pwd import re import select import signal import stat import struct import sys import termios import textwrap import threading import time import traceback import tty import warnings import weakref from asyncio import Queue as AQueue from contextlib import contextmanager from functools import partial from io import BytesIO, StringIO, UnsupportedOperation from io import open as fdopen from locale import getpreferredencoding from queue import Empty, Queue from shlex import quote as shlex_quote from types import GeneratorType, ModuleType from typing import Any, Dict, Type, Union __project_url__ = "https://github.com/amoffat/sh" TEE_STDOUT = {True, "out", 1} TEE_STDERR = {"err", 2} DEFAULT_ENCODING = getpreferredencoding() or "UTF-8" IS_MACOS = platform.system() in ("AIX", "Darwin") SH_LOGGER_NAME = __name__ # normally i would hate this idea of using a global to signify whether we are # running tests, because it breaks the assumption that what is running in the # tests is what will run live, but we ONLY use this in a place that has no # serious side-effects that could change anything. as long as we do that, it # should be ok RUNNING_TESTS = bool(int(os.environ.get("SH_TESTS_RUNNING", "0"))) FORCE_USE_SELECT = bool(int(os.environ.get("SH_TESTS_USE_SELECT", "0"))) # a re-entrant lock for pushd. this way, multiple threads that happen to use # pushd will all see the current working directory for the duration of the # with-context PUSHD_LOCK = threading.RLock() def get_num_args(fn): return len(inspect.getfullargspec(fn).args) _unicode_methods = set(dir("")) HAS_POLL = hasattr(select, "poll") POLLER_EVENT_READ = 1 POLLER_EVENT_WRITE = 2 POLLER_EVENT_HUP = 4 POLLER_EVENT_ERROR = 8 class PollPoller: def __init__(self): self._poll = select.poll() # file descriptor <-> file object bidirectional maps self.fd_lookup = {} self.fo_lookup = {} def __nonzero__(self): return len(self.fd_lookup) != 0 def __len__(self): return len(self.fd_lookup) def _set_fileobject(self, f): if hasattr(f, "fileno"): fd = f.fileno() self.fd_lookup[fd] = f self.fo_lookup[f] = fd else: self.fd_lookup[f] = f self.fo_lookup[f] = f def _remove_fileobject(self, f): if hasattr(f, "fileno"): fd = f.fileno() del self.fd_lookup[fd] del self.fo_lookup[f] else: del self.fd_lookup[f] del self.fo_lookup[f] def _get_file_descriptor(self, f): return self.fo_lookup.get(f) def _get_file_object(self, fd): return self.fd_lookup.get(fd) def _register(self, f, events): # f can be a file descriptor or file object self._set_fileobject(f) fd = self._get_file_descriptor(f) self._poll.register(fd, events) def register_read(self, f): self._register(f, select.POLLIN | select.POLLPRI) def register_write(self, f): self._register(f, select.POLLOUT) def register_error(self, f): self._register(f, select.POLLERR | select.POLLHUP | select.POLLNVAL) def unregister(self, f): fd = self._get_file_descriptor(f) self._poll.unregister(fd) self._remove_fileobject(f) def poll(self, timeout): if timeout is not None: # convert from seconds to milliseconds timeout *= 1000 changes = self._poll.poll(timeout) results = [] for fd, events in changes: f = self._get_file_object(fd) if events & (select.POLLIN | select.POLLPRI): results.append((f, POLLER_EVENT_READ)) elif events & select.POLLOUT: results.append((f, POLLER_EVENT_WRITE)) elif events & select.POLLHUP: results.append((f, POLLER_EVENT_HUP)) elif events & (select.POLLERR | select.POLLNVAL): results.append((f, POLLER_EVENT_ERROR)) return results class SelectPoller: def __init__(self): self.rlist = [] self.wlist = [] self.xlist = [] def __nonzero__(self): return len(self.rlist) + len(self.wlist) + len(self.xlist) != 0 def __len__(self): return len(self.rlist) + len(self.wlist) + len(self.xlist) @staticmethod def _register(f, events): if f not in events: events.append(f) @staticmethod def _unregister(f, events): if f in events: events.remove(f) def register_read(self, f): self._register(f, self.rlist) def register_write(self, f): self._register(f, self.wlist) def register_error(self, f): self._register(f, self.xlist) def unregister(self, f): self._unregister(f, self.rlist) self._unregister(f, self.wlist) self._unregister(f, self.xlist) def poll(self, timeout): _in, _out, _err = select.select(self.rlist, self.wlist, self.xlist, timeout) results = [] for f in _in: results.append((f, POLLER_EVENT_READ)) for f in _out: results.append((f, POLLER_EVENT_WRITE)) for f in _err: results.append((f, POLLER_EVENT_ERROR)) return results # here we use an use a poller interface that transparently selects the most # capable poller (out of either select.select or select.poll). this was added # by zhangyafeikimi when he discovered that if the fds created internally by sh # numbered > 1024, select.select failed (a limitation of select.select). this # can happen if your script opens a lot of files Poller: Union[Type[SelectPoller], Type[PollPoller]] = SelectPoller if HAS_POLL and not FORCE_USE_SELECT: Poller = PollPoller class ForkException(Exception): def __init__(self, orig_exc): msg = f""" Original exception: =================== {textwrap.indent(orig_exc, " ")} """ Exception.__init__(self, msg) class ErrorReturnCodeMeta(type): """a metaclass which provides the ability for an ErrorReturnCode (or derived) instance, imported from one sh module, to be considered the subclass of ErrorReturnCode from another module. this is mostly necessary in the tests, where we do assertRaises, but the ErrorReturnCode that the program we're testing throws may not be the same class that we pass to assertRaises """ def __subclasscheck__(self, o): other_bases = {b.__name__ for b in o.__bases__} return self.__name__ in other_bases or o.__name__ == self.__name__ class ErrorReturnCode(Exception): __metaclass__ = ErrorReturnCodeMeta """ base class for all exceptions as a result of a command's exit status being deemed an error. this base class is dynamically subclassed into derived classes with the format: ErrorReturnCode_NNN where NNN is the exit code number. the reason for this is it reduces boiler plate code when testing error return codes: try: some_cmd() except ErrorReturnCode_12: print("couldn't do X") vs: try: some_cmd() except ErrorReturnCode as e: if e.exit_code == 12: print("couldn't do X") it's not much of a savings, but i believe it makes the code easier to read """ truncate_cap = 750 def __reduce__(self): return self.__class__, (self.full_cmd, self.stdout, self.stderr, self.truncate) def __init__(self, full_cmd, stdout, stderr, truncate=True): self.exit_code = self.exit_code # makes pylint happy self.full_cmd = full_cmd self.stdout = stdout self.stderr = stderr self.truncate = truncate exc_stdout = self.stdout if truncate: exc_stdout = exc_stdout[: self.truncate_cap] out_delta = len(self.stdout) - len(exc_stdout) if out_delta: exc_stdout += (f"... ({out_delta} more, please see e.stdout)").encode() exc_stderr = self.stderr if truncate: exc_stderr = exc_stderr[: self.truncate_cap] err_delta = len(self.stderr) - len(exc_stderr) if err_delta: exc_stderr += (f"... ({err_delta} more, please see e.stderr)").encode() msg = ( f"\n\n RAN: {self.full_cmd}" f"\n\n STDOUT:\n{exc_stdout.decode(DEFAULT_ENCODING, 'replace')}" f"\n\n STDERR:\n{exc_stderr.decode(DEFAULT_ENCODING, 'replace')}" ) super().__init__(msg) class SignalException(ErrorReturnCode): pass class TimeoutException(Exception): """the exception thrown when a command is killed because a specified timeout (via _timeout or .wait(timeout)) was hit""" def __init__(self, exit_code, full_cmd): self.exit_code = exit_code self.full_cmd = full_cmd super(Exception, self).__init__() SIGNALS_THAT_SHOULD_THROW_EXCEPTION = { signal.SIGABRT, signal.SIGBUS, signal.SIGFPE, signal.SIGILL, signal.SIGINT, signal.SIGKILL, signal.SIGPIPE, signal.SIGQUIT, signal.SIGSEGV, signal.SIGTERM, signal.SIGSYS, } # we subclass AttributeError because: # https://github.com/ipython/ipython/issues/2577 # https://github.com/amoffat/sh/issues/97#issuecomment-10610629 class CommandNotFound(AttributeError): pass rc_exc_regex = re.compile(r"(ErrorReturnCode|SignalException)_((\d+)|SIG[a-zA-Z]+)") rc_exc_cache: Dict[str, Type[ErrorReturnCode]] = {} SIGNAL_MAPPING = { v: k for k, v in signal.__dict__.items() if re.match(r"SIG[a-zA-Z]+", k) } def get_exc_from_name(name): """takes an exception name, like: ErrorReturnCode_1 SignalException_9 SignalException_SIGHUP and returns the corresponding exception. this is primarily used for importing exceptions from sh into user code, for instance, to capture those exceptions""" exc = None try: return rc_exc_cache[name] except KeyError: m = rc_exc_regex.match(name) if m: base = m.group(1) rc_or_sig_name = m.group(2) if base == "SignalException": try: rc = -int(rc_or_sig_name) except ValueError: rc = -getattr(signal, rc_or_sig_name) else: rc = int(rc_or_sig_name) exc = get_rc_exc(rc) return exc def get_rc_exc(rc): """takes a exit code or negative signal number and produces an exception that corresponds to that return code. positive return codes yield ErrorReturnCode exception, negative return codes yield SignalException we also cache the generated exception so that only one signal of that type exists, preserving identity""" try: return rc_exc_cache[rc] except KeyError: pass if rc >= 0: name = f"ErrorReturnCode_{rc}" base = ErrorReturnCode else: name = f"SignalException_{SIGNAL_MAPPING[abs(rc)]}" base = SignalException exc = ErrorReturnCodeMeta(name, (base,), {"exit_code": rc}) rc_exc_cache[rc] = exc return exc # we monkey patch glob. i'm normally generally against monkey patching, but i # decided to do this really un-intrusive patch because we need a way to detect # if a list that we pass into an sh command was generated from glob. the reason # being that glob returns an empty list if a pattern is not found, and so # commands will treat the empty list as no arguments, which can be a problem, # ie: # # ls(glob("*.ojfawe")) # # ^ will show the contents of your home directory, because it's essentially # running ls([]) which, as a process, is just "ls". # # so we subclass list and monkey patch the glob function. nobody should be the # wiser, but we'll have results that we can make some determinations on _old_glob = glob_module.glob class GlobResults(list): def __init__(self, path, results): self.path = path list.__init__(self, results) def glob(path, *args, **kwargs): expanded = GlobResults(path, _old_glob(path, *args, **kwargs)) return expanded glob_module.glob = glob # type: ignore def canonicalize(path): return os.path.abspath(os.path.expanduser(path)) def _which(program, paths=None): """takes a program name or full path, plus an optional collection of search paths, and returns the full path of the requested executable. if paths is specified, it is the entire list of search paths, and the PATH env is not used at all. otherwise, PATH env is used to look for the program""" def is_exe(file_path): return ( os.path.exists(file_path) and os.access(file_path, os.X_OK) and os.path.isfile(os.path.realpath(file_path)) ) found_path = None fpath, fname = os.path.split(program) # if there's a path component, then we've specified a path to the program, # and we should just test if that program is executable. if it is, return if fpath: program = canonicalize(program) if is_exe(program): found_path = program # otherwise, we've just passed in the program name, and we need to search # the paths to find where it actually lives else: paths_to_search = [] if isinstance(paths, (tuple, list)): paths_to_search.extend(paths) else: env_paths = os.environ.get("PATH", "").split(os.pathsep) paths_to_search.extend(env_paths) for path in paths_to_search: exe_file = os.path.join(canonicalize(path), program) if is_exe(exe_file): found_path = exe_file break return found_path def resolve_command_path(program): path = _which(program) if not path: # our actual command might have a dash in it, but we can't call # that from python (we have to use underscores), so we'll check # if a dash version of our underscore command exists and use that # if it does if "_" in program: path = _which(program.replace("_", "-")) if not path: return None return path def resolve_command(name, command_cls, baked_args=None): path = resolve_command_path(name) cmd = None if path: cmd = command_cls(path) if baked_args: cmd = cmd.bake(**baked_args) return cmd class Logger: """provides a memory-inexpensive logger. a gotcha about python's builtin logger is that logger objects are never garbage collected. if you create a thousand loggers with unique names, they'll sit there in memory until your script is done. with sh, it's easy to create loggers with unique names if we want our loggers to include our command arguments. for example, these are all unique loggers: ls -l ls -l /tmp ls /tmp so instead of creating unique loggers, and without sacrificing logging output, we use this class, which maintains as part of its state, the logging "context", which will be the very unique name. this allows us to get a logger with a very general name, eg: "command", and have a unique name appended to it via the context, eg: "ls -l /tmp" """ def __init__(self, name, context=None): self.name = name self.log = logging.getLogger(f"{SH_LOGGER_NAME}.{name}") self.context = self.sanitize_context(context) def _format_msg(self, msg, *a): if self.context: msg = f"{self.context}: {msg}" return msg % a @staticmethod def sanitize_context(context): if context: context = context.replace("%", "%%") return context or "" def get_child(self, name, context): new_name = self.name + "." + name new_context = self.context + "." + context return Logger(new_name, new_context) def info(self, msg, *a): self.log.info(self._format_msg(msg, *a)) def debug(self, msg, *a): self.log.debug(self._format_msg(msg, *a)) def error(self, msg, *a): self.log.error(self._format_msg(msg, *a)) def exception(self, msg, *a): self.log.exception(self._format_msg(msg, *a)) def default_logger_str(cmd, call_args, pid=None): if pid: s = f"" else: s = f"" return s class RunningCommand: """this represents an executing Command object. it is returned as the result of __call__() being executed on a Command instance. this creates a reference to a OProc instance, which is a low-level wrapper around the process that was exec'd this is the class that gets manipulated the most by user code, and so it implements various convenience methods and logical mechanisms for the underlying process. for example, if a user tries to access a backgrounded-process's stdout/err, the RunningCommand object is smart enough to know to wait() on the process to finish first. and when the process finishes, RunningCommand is smart enough to translate exit codes to exceptions.""" # these are attributes that we allow to pass through to OProc _OProc_attr_allowlist = { "signal", "terminate", "kill", "kill_group", "signal_group", "pid", "sid", "pgid", "ctty", "input_thread_exc", "output_thread_exc", "bg_thread_exc", } def __init__(self, cmd, call_args, stdin, stdout, stderr): # self.ran is used for auditing what actually ran. for example, in # exceptions, or if you just want to know what was ran after the # command ran self.ran = " ".join([shlex_quote(str(arg)) for arg in cmd]) self.call_args = call_args self.cmd = cmd self.process = None self._waited_until_completion = False should_wait = True spawn_process = True # if we're using an async for loop on this object, we need to put the underlying # iterable in no-block mode. however, we will only know if we're using an async # for loop after this object is constructed. so we'll set it to False now, but # then later set it to True if we need it self._force_noblock_iter = False # this event is used when we want to `await` a RunningCommand. see how it gets # used in self.__await__ try: asyncio.get_running_loop() except RuntimeError: self.aio_output_complete = None else: self.aio_output_complete = asyncio.Event() # this is used to track if we've already raised StopIteration, and if we # have, raise it immediately again if the user tries to call next() on # us. https://github.com/amoffat/sh/issues/273 self._stopped_iteration = False # with contexts shouldn't run at all yet, they prepend # to every command in the context if call_args["with"]: spawn_process = False get_prepend_stack().append(self) if call_args["piped"] or call_args["iter"] or call_args["iter_noblock"]: should_wait = False if call_args["async"]: should_wait = False # we're running in the background, return self and let us lazily # evaluate if call_args["bg"]: should_wait = False # redirection if call_args["err_to_out"]: stderr = OProc.STDOUT done_callback = call_args["done"] if done_callback: call_args["done"] = partial(done_callback, self) # set up which stream should write to the pipe # TODO, make pipe None by default and limit the size of the Queue # in oproc.OProc pipe = OProc.STDOUT if call_args["iter"] == "out" or call_args["iter"] is True: pipe = OProc.STDOUT elif call_args["iter"] == "err": pipe = OProc.STDERR if call_args["iter_noblock"] == "out" or call_args["iter_noblock"] is True: pipe = OProc.STDOUT elif call_args["iter_noblock"] == "err": pipe = OProc.STDERR # there's currently only one case where we wouldn't spawn a child # process, and that's if we're using a with-context with our command self._spawned_and_waited = False if spawn_process: log_str_factory = call_args["log_msg"] or default_logger_str logger_str = log_str_factory(self.ran, call_args) self.log = Logger("command", logger_str) self.log.debug("starting process") if should_wait: self._spawned_and_waited = True # this lock is needed because of a race condition where a background # thread, created in the OProc constructor, may try to access # self.process, but it has not been assigned yet process_assign_lock = threading.Lock() with process_assign_lock: self.process = OProc( self, self.log, cmd, stdin, stdout, stderr, self.call_args, pipe, process_assign_lock, ) logger_str = log_str_factory(self.ran, call_args, self.process.pid) self.log.context = self.log.sanitize_context(logger_str) self.log.info("process started") if should_wait: self.wait() def wait(self, timeout=None): """waits for the running command to finish. this is called on all running commands, eventually, except for ones that run in the background if timeout is a number, it is the number of seconds to wait for the process to resolve. otherwise block on wait. this function can raise a TimeoutException, either because of a `_timeout` on the command itself as it was launched, or because of a timeout passed into this method. """ if not self._waited_until_completion: # if we've been given a timeout, we need to poll is_alive() if timeout is not None: waited_for = 0 sleep_amt = 0.1 alive = False exit_code = None if timeout < 0: raise RuntimeError("timeout cannot be negative") # while we still have time to wait, run this loop # notice that alive and exit_code are only defined in this loop, but # the loop is also guaranteed to run, defining them, given the # constraints that timeout is non-negative while waited_for <= timeout: alive, exit_code = self.process.is_alive() # if we're alive, we need to wait some more, but let's sleep # before we poll again if alive: time.sleep(sleep_amt) waited_for += sleep_amt # but if we're not alive, we're done waiting else: break # if we've made it this far, and we're still alive, then it means we # timed out waiting if alive: raise TimeoutException(None, self.ran) # if we didn't time out, we fall through and let the rest of the code # handle exit_code. notice that we set _waited_until_completion here, # only if we didn't time out. this allows us to re-wait again on # timeout, if we catch the TimeoutException in the parent frame self._waited_until_completion = True else: exit_code = self.process.wait() self._waited_until_completion = True if self.process.timed_out: # if we timed out, our exit code represents a signal, which is # negative, so let's make it positive to store in our # TimeoutException raise TimeoutException(-exit_code, self.ran) else: self.handle_command_exit_code(exit_code) # if an iterable command is using an instance of OProc for its stdin, # wait on it. the process is probably set to "piped", which means it # won't be waited on, which means exceptions won't propagate up to the # main thread. this allows them to bubble up if self.process._stdin_process: self.process._stdin_process.command.wait() self.log.debug("process completed") return self def is_alive(self): """returns whether or not we're still alive. this call has side-effects on OProc""" return self.process.is_alive()[0] def handle_command_exit_code(self, code): """here we determine if we had an exception, or an error code that we weren't expecting to see. if we did, we create and raise an exception """ ca = self.call_args exc_class = get_exc_exit_code_would_raise(code, ca["ok_code"], ca["piped"]) if exc_class: exc = exc_class( self.ran, self.process.stdout, self.process.stderr, ca["truncate_exc"] ) raise exc @property def stdout(self): self.wait() return self.process.stdout @property def stderr(self): self.wait() return self.process.stderr @property def exit_code(self): self.wait() return self.process.exit_code def __len__(self): return len(str(self)) def __enter__(self): """we don't actually do anything here because anything that should have been done would have been done in the Command.__call__ call. essentially all that has to happen is the command be pushed on the prepend stack.""" pass def __iter__(self): return self def __next__(self): """allow us to iterate over the output of our command""" if self._stopped_iteration: raise StopIteration() pq = self.process._pipe_queue # the idea with this is, if we're using regular `_iter` (non-asyncio), then we # want to have blocking be True when we read from the pipe queue, so our cpu # doesn't spin too fast. however, if we *are* using asyncio (an async for loop), # then we want non-blocking pipe queue reads, because we'll do an asyncio.sleep, # in the coroutine that is doing the iteration, this way coroutines have better # yielding (see queue_connector in __aiter__). block_pq_read = not self._force_noblock_iter # we do this because if get blocks, we can't catch a KeyboardInterrupt # so the slight timeout allows for that. while True: try: chunk = pq.get(block_pq_read, self.call_args["iter_poll_time"]) except Empty: if self.call_args["iter_noblock"] or self._force_noblock_iter: return errno.EWOULDBLOCK else: if chunk is None: self.wait() self._stopped_iteration = True raise StopIteration() try: return chunk.decode( self.call_args["encoding"], self.call_args["decode_errors"] ) except UnicodeDecodeError: return chunk def __await__(self): async def wait_for_completion(): await self.aio_output_complete.wait() if self.call_args["return_cmd"]: # We know the command has completed already, # but need to catch exceptions self.wait() return self else: return str(self) return wait_for_completion().__await__() def __aiter__(self): # maxsize is critical to making sure our queue_connector function below yields # when it awaits _aio_queue.put(chunk). if we didn't have a maxsize, our loop # would happily iterate through `chunk in self` and put onto the queue without # any blocking, and therefore no yielding, which would prevent other coroutines # from running. self._aio_queue = AQueue(maxsize=1) self._force_noblock_iter = True # the sole purpose of this coroutine is to connect our pipe_queue (which is # being populated by a thread) to an asyncio-friendly queue. then, in __anext__, # we can iterate over that asyncio queue. async def queue_connector(): try: # this will spin as fast as possible if there's no data to read, # thanks to self._force_noblock_iter. so we sleep below. for chunk in self: if chunk == errno.EWOULDBLOCK: # let us have better coroutine yielding. await asyncio.sleep(0.01) else: await self._aio_queue.put(chunk) finally: await self._aio_queue.put(None) task = asyncio.create_task(queue_connector()) self._aio_task = task return self async def __anext__(self): chunk = await self._aio_queue.get() if chunk is not None: return chunk else: exc = self._aio_task.exception() if exc is not None: raise exc raise StopAsyncIteration def __exit__(self, exc_type, exc_val, exc_tb): if self.call_args["with"] and get_prepend_stack(): get_prepend_stack().pop() def __str__(self): if self.process and self.stdout: return self.stdout.decode( self.call_args["encoding"], self.call_args["decode_errors"] ) return "" def __eq__(self, other): return id(self) == id(other) def __contains__(self, item): return item in str(self) def __getattr__(self, p): # let these three attributes pass through to the OProc object if p in self._OProc_attr_allowlist: if self.process: return getattr(self.process, p) else: raise AttributeError # see if strings have what we're looking for if p in _unicode_methods: return getattr(str(self), p) raise AttributeError def __repr__(self): try: return str(self) except UnicodeDecodeError: if self.process: if self.stdout: return repr(self.stdout) return repr("") def __long__(self): return int(str(self).strip()) def __float__(self): return float(str(self).strip()) def __int__(self): return int(str(self).strip()) def output_redirect_is_filename(out): return isinstance(out, str) or hasattr(out, "__fspath__") def get_prepend_stack(): tl = Command.thread_local if not hasattr(tl, "_prepend_stack"): tl._prepend_stack = [] return tl._prepend_stack def special_kwarg_validator(passed_kwargs, merged_kwargs, invalid_list): s1 = set(passed_kwargs.keys()) invalid_args = [] for elem in invalid_list: if callable(elem): fn = elem ret = fn(passed_kwargs, merged_kwargs) invalid_args.extend(ret) else: elem, error_msg = elem if s1.issuperset(elem): invalid_args.append((elem, error_msg)) return invalid_args def get_fileno(ob): # in py2, this will return None. in py3, it will return an method that # raises when called fileno_meth = getattr(ob, "fileno", None) fileno = None if fileno_meth: # py3 StringIO objects will report a fileno, but calling it will raise # an exception try: fileno = fileno_meth() except UnsupportedOperation: pass elif isinstance(ob, (int,)) and ob >= 0: fileno = ob return fileno def ob_is_fd_based(ob): return get_fileno(ob) is not None def ob_is_tty(ob): """checks if an object (like a file-like object) is a tty.""" fileno = get_fileno(ob) is_tty = False if fileno is not None: is_tty = os.isatty(fileno) return is_tty def ob_is_pipe(ob): fileno = get_fileno(ob) is_pipe = False if fileno: fd_stat = os.fstat(fileno) is_pipe = stat.S_ISFIFO(fd_stat.st_mode) return is_pipe def output_iterator_validator(passed_kwargs, merged_kwargs): invalid = [] if passed_kwargs.get("no_out") and passed_kwargs.get("iter") in (True, "out"): error = "You cannot iterate over output if there is no output" invalid.append((("no_out", "iter"), error)) return invalid def tty_in_validator(passed_kwargs, merged_kwargs): # here we'll validate that people aren't randomly shotgun-debugging different tty # options and hoping that they'll work, without understanding what they do pairs = (("tty_in", "in"), ("tty_out", "out")) invalid = [] for tty_type, std in pairs: if tty_type in passed_kwargs and ob_is_tty(passed_kwargs.get(std, None)): error = ( f"`_{std}` is a TTY already, so so it doesn't make sense to set up a" f" TTY with `_{tty_type}`" ) invalid.append(((tty_type, std), error)) # if unify_ttys is set, then both tty_in and tty_out must both be True if merged_kwargs["unify_ttys"] and not ( merged_kwargs["tty_in"] and merged_kwargs["tty_out"] ): invalid.append( ( ("unify_ttys", "tty_in", "tty_out"), "`_tty_in` and `_tty_out` must both be True if `_unify_ttys` is True", ) ) return invalid def fg_validator(passed_kwargs, merged_kwargs): """fg is not valid with basically every other option""" invalid = [] msg = """\ _fg is invalid with nearly every other option, see warning and workaround here: https://sh.readthedocs.io/en/latest/sections/special_arguments.html#fg""" allowlist = {"env", "fg", "cwd", "ok_code"} offending = set(passed_kwargs.keys()) - allowlist if "fg" in passed_kwargs and passed_kwargs["fg"] and offending: invalid.append(("fg", msg)) return invalid def bufsize_validator(passed_kwargs, merged_kwargs): """a validator to prevent a user from saying that they want custom buffering when they're using an in/out object that will be os.dup'ed to the process, and has its own buffering. an example is a pipe or a tty. it doesn't make sense to tell them to have a custom buffering, since the os controls this.""" invalid = [] in_ob = passed_kwargs.get("in", None) out_ob = passed_kwargs.get("out", None) in_buf = passed_kwargs.get("in_bufsize", None) out_buf = passed_kwargs.get("out_bufsize", None) in_no_buf = ob_is_fd_based(in_ob) out_no_buf = ob_is_fd_based(out_ob) err = "Can't specify an {target} bufsize if the {target} target is a pipe or TTY" if in_no_buf and in_buf is not None: invalid.append((("in", "in_bufsize"), err.format(target="in"))) if out_no_buf and out_buf is not None: invalid.append((("out", "out_bufsize"), err.format(target="out"))) return invalid def env_validator(passed_kwargs, merged_kwargs): """a validator to check that env is a dictionary and that all environment variable keys and values are strings. Otherwise, we would exit with a confusing exit code 255.""" invalid = [] env = passed_kwargs.get("env", None) if env is None: return invalid if not isinstance(env, Mapping): invalid.append(("env", f"env must be dict-like. Got {env!r}")) return invalid for k, v in passed_kwargs["env"].items(): if not isinstance(k, str): invalid.append(("env", f"env key {k!r} must be a str")) if not isinstance(v, str): invalid.append(("env", f"value {v!r} of env key {k!r} must be a str")) return invalid class Command: """represents an un-run system program, like "ls" or "cd". because it represents the program itself (and not a running instance of it), it should hold very little state. in fact, the only state it does hold is baked arguments. when a Command object is called, the result that is returned is a RunningCommand object, which represents the Command put into an execution state.""" thread_local = threading.local() RunningCommandCls = RunningCommand _call_args: Dict[str, Any] = { "fg": False, # run command in foreground # run a command in the background. commands run in the background # ignore SIGHUP and do not automatically exit when the parent process # ends "bg": False, # automatically report exceptions for background commands "bg_exc": True, "with": False, # prepend the command to every command after it "in": None, "out": None, # redirect STDOUT "err": None, # redirect STDERR "err_to_out": None, # redirect STDERR to STDOUT # stdin buffer size # 1 for line, 0 for unbuffered, any other number for that amount "in_bufsize": 0, # stdout buffer size, same values as above "out_bufsize": 1, "err_bufsize": 1, # this is how big the output buffers will be for stdout and stderr. # this is essentially how much output they will store from the process. # we use a deque, so if it overflows past this amount, the first items # get pushed off as each new item gets added. # # NOTICE # this is not a *BYTE* size, this is a *CHUNK* size...meaning, that if # you're buffering out/err at 1024 bytes, the internal buffer size will # be "internal_bufsize" CHUNKS of 1024 bytes "internal_bufsize": 3 * 1024**2, "env": None, "piped": None, "iter": None, "iter_noblock": None, # the amount of time to sleep between polling for the iter output queue "iter_poll_time": 0.1, "ok_code": 0, "cwd": None, # the separator delimiting between a long-argument's name and its value # setting this to None will cause name and value to be two separate # arguments, like for short options # for example, --arg=derp, '=' is the long_sep "long_sep": "=", # the prefix used for long arguments "long_prefix": "--", # this is for programs that expect their input to be from a terminal. # ssh is one of those programs "tty_in": False, "tty_out": True, "unify_ttys": False, "encoding": DEFAULT_ENCODING, "decode_errors": "strict", # how long the process should run before it is auto-killed "timeout": None, "timeout_signal": signal.SIGKILL, # TODO write some docs on "long-running processes" # these control whether or not stdout/err will get aggregated together # as the process runs. this has memory usage implications, so sometimes # with long-running processes with a lot of data, it makes sense to # set these to true "no_out": False, "no_err": False, "no_pipe": False, # if any redirection is used for stdout or stderr, internal buffering # of that data is not stored. this forces it to be stored, as if # the output is being T'd to both the redirected destination and our # internal buffers "tee": None, # will be called when a process terminates regardless of exception "done": None, # a tuple (rows, columns) of the desired size of both the stdout and # stdin ttys, if ttys are being used "tty_size": (24, 80), # whether or not our exceptions should be truncated "truncate_exc": True, # a function to call after the child forks but before the process execs "preexec_fn": None, # UID to set after forking. Requires root privileges. Not supported on # Windows. "uid": None, # put the forked process in its own process session? "new_session": False, # put the forked process in its own process group? "new_group": False, # pre-process args passed into __call__. only really useful when used # in .bake() "arg_preprocess": None, # a callable that produces a log message from an argument tuple of the # command and the args "log_msg": None, # whether or not to close all inherited fds. typically, this should be True, # as inheriting fds can be a security vulnerability "close_fds": True, # a allowlist of the integer fds to pass through to the child process. setting # this forces close_fds to be True "pass_fds": set(), # return an instance of RunningCommand always. if this isn't True, then # sometimes we may return just a plain unicode string "return_cmd": False, "async": False, } # this is a collection of validators to make sure the special kwargs make # sense _kwarg_validators = ( (("err", "err_to_out"), "Stderr is already being redirected"), (("piped", "iter"), "You cannot iterate when this command is being piped"), ( ("piped", "no_pipe"), "Using a pipe doesn't make sense if you've disabled the pipe", ), output_iterator_validator, (("close_fds", "pass_fds"), "Passing `pass_fds` forces `close_fds` to be True"), tty_in_validator, bufsize_validator, env_validator, fg_validator, ) def __init__(self, path, search_paths=None): found = _which(path, search_paths) self._path = "" # is the command baked (aka, partially applied)? self._partial = False self._partial_baked_args = [] self._partial_call_args = {} # bugfix for functools.wraps. issue #121 self.__name__ = str(self) if not found: raise CommandNotFound(path) # the reason why we set the values early in the constructor, and again # here, is for people who have tools that inspect the stack on # exception. if CommandNotFound is raised, we need self._path and the # other attributes to be set correctly, so repr() works when they're # inspecting the stack. issue #304 self._path = found self.__name__ = str(self) def __getattribute__(self, name): # convenience get_attr = partial(object.__getattribute__, self) val = None if name.startswith("_"): val = get_attr(name) elif name == "bake": val = get_attr("bake") # here we have a way of getting past shadowed subcommands. for example, # if "git bake" was a thing, we wouldn't be able to do `git.bake()` # because `.bake()` is already a method. so we allow `git.bake_()` elif name.endswith("_"): name = name[:-1] if val is None: val = get_attr("bake")(name) return val @classmethod def _extract_call_args(cls, kwargs): """takes kwargs that were passed to a command's __call__ and extracts out the special keyword arguments, we return a tuple of special keyword args, and kwargs that will go to the exec'ed command""" kwargs = kwargs.copy() call_args = {} for parg, default in cls._call_args.items(): key = "_" + parg if key in kwargs: call_args[parg] = kwargs[key] del kwargs[key] merged_args = cls._call_args.copy() merged_args.update(call_args) invalid_kwargs = special_kwarg_validator( call_args, merged_args, cls._kwarg_validators ) if invalid_kwargs: exc_msg = [] for kwarg, error_msg in invalid_kwargs: exc_msg.append(f" {kwarg!r}: {error_msg}") exc_msg = "\n".join(exc_msg) raise TypeError(f"Invalid special arguments:\n\n{exc_msg}\n") return call_args, kwargs def bake(self, *args, **kwargs): """returns a new Command object after baking(freezing) the given command arguments which are used automatically when its exec'ed special keyword arguments can be temporary baked and additionally be overridden in __call__ or in subsequent bakes (basically setting defaults)""" # construct the base Command fn = type(self)(self._path) fn._partial = True call_args, kwargs = self._extract_call_args(kwargs) fn._partial_call_args.update(self._partial_call_args) fn._partial_call_args.update(call_args) fn._partial_baked_args.extend(self._partial_baked_args) sep = call_args.get("long_sep", self._call_args["long_sep"]) prefix = call_args.get("long_prefix", self._call_args["long_prefix"]) fn._partial_baked_args.extend(compile_args(args, kwargs, sep, prefix)) return fn def __str__(self): if not self._partial_baked_args: return self._path baked_args = " ".join(shlex_quote(arg) for arg in self._partial_baked_args) return f"{self._path} {baked_args}" def __eq__(self, other): return str(self) == str(other) def __repr__(self): return f"" def __enter__(self): self(_with=True) def __exit__(self, exc_type, exc_val, exc_tb): get_prepend_stack().pop() def __call__(self, *args, **kwargs): kwargs = kwargs.copy() args = list(args) # this will hold our final command, including arguments, that will be # exec'ed cmd = [] # this will hold a complete mapping of all our special keyword arguments # and their values call_args = self.__class__._call_args.copy() # aggregate any 'with' contexts for prepend in get_prepend_stack(): pcall_args = prepend.call_args.copy() # don't pass the 'with' call arg pcall_args.pop("with", None) call_args.update(pcall_args) # we do not prepend commands used as a 'with' context as they will # be prepended to any nested commands if not kwargs.get("_with", False): cmd.extend(prepend.cmd) cmd.append(self._path) # do we have an argument pre-processor? if so, run it. we need to do # this early, so that args, kwargs are accurate preprocessor = self._partial_call_args.get("arg_preprocess", None) if preprocessor: args, kwargs = preprocessor(args, kwargs) # here we extract the special kwargs and override any # special kwargs from the possibly baked command extracted_call_args, kwargs = self._extract_call_args(kwargs) call_args.update(self._partial_call_args) call_args.update(extracted_call_args) # handle a None. this is added back only to not break the api in the # 1.* version. TODO remove this in 2.0, as "ok_code", if specified, # should always be a definitive value or list of values, and None is # ambiguous if call_args["ok_code"] is None: call_args["ok_code"] = 0 if not getattr(call_args["ok_code"], "__iter__", None): call_args["ok_code"] = [call_args["ok_code"]] # determine what our real STDIN is. is it something explicitly passed into # _in? stdin = call_args["in"] # now that we have our stdin, let's figure out how we should handle it if isinstance(stdin, RunningCommand): if stdin.call_args["piped"]: stdin = stdin.process else: stdin = stdin.process._pipe_queue processed_args = compile_args( args, kwargs, call_args["long_sep"], call_args["long_prefix"] ) # makes sure our arguments are broken up correctly split_args = self._partial_baked_args + processed_args final_args = split_args cmd.extend(final_args) # if we're running in foreground mode, we need to completely bypass # launching a RunningCommand and OProc and just do a spawn if call_args["fg"]: cwd = call_args["cwd"] or os.getcwd() with pushd(cwd): if call_args["env"] is None: exit_code = os.spawnv(os.P_WAIT, cmd[0], cmd) else: exit_code = os.spawnve(os.P_WAIT, cmd[0], cmd, call_args["env"]) exc_class = get_exc_exit_code_would_raise( exit_code, call_args["ok_code"], call_args["piped"] ) if exc_class: ran = " ".join(cmd) exc = exc_class(ran, b"", b"", call_args["truncate_exc"]) raise exc return None # stdout redirection stdout = call_args["out"] if output_redirect_is_filename(stdout): stdout = open(str(stdout), "wb") # stderr redirection stderr = call_args["err"] if output_redirect_is_filename(stderr): stderr = open(str(stderr), "wb") rc = self.__class__.RunningCommandCls(cmd, call_args, stdin, stdout, stderr) if rc._spawned_and_waited and not call_args["return_cmd"]: return str(rc) else: return rc def compile_args(a, kwargs, sep, prefix): """takes args and kwargs, as they were passed into the command instance being executed with __call__, and compose them into a flat list that will eventually be fed into exec. example: with this call: sh.ls("-l", "/tmp", color="never") this function receives args = ['-l', '/tmp'] kwargs = {'color': 'never'} and produces ['-l', '/tmp', '--color=geneticnever'] """ processed_args = [] # aggregate positional args for arg in a: if isinstance(arg, (list, tuple)): if isinstance(arg, GlobResults) and not arg: arg = [arg.path] for sub_arg in arg: processed_args.append(sub_arg) elif isinstance(arg, dict): processed_args += _aggregate_keywords(arg, sep, prefix, raw=True) # see https://github.com/amoffat/sh/issues/522 elif arg is None or arg is False: pass else: processed_args.append(str(arg)) # aggregate the keyword arguments processed_args += _aggregate_keywords(kwargs, sep, prefix) return processed_args def _aggregate_keywords(keywords, sep, prefix, raw=False): """take our keyword arguments, and a separator, and compose the list of flat long (and short) arguments. example {'color': 'never', 't': True, 'something': True} with sep '=' becomes ['--color=never', '-t', '--something'] the `raw` argument indicates whether or not we should leave the argument name alone, or whether we should replace "_" with "-". if we pass in a dictionary, like this: sh.command({"some_option": 12}) then `raw` gets set to True, because we want to leave the key as-is, to produce: ['--some_option=12'] but if we just use a command's kwargs, `raw` is False, which means this: sh.command(some_option=12) becomes: ['--some-option=12'] essentially, using kwargs is a convenience, but it lacks the ability to put a '-' in the name, so we do the replacement of '_' to '-' for you. but when you really don't want that to happen, you should use a dictionary instead with the exact names you want """ processed = [] for k, maybe_list_of_v in keywords.items(): # turn our value(s) into a list of values so that we can process them # all individually under the same key list_of_v = [maybe_list_of_v] if isinstance(maybe_list_of_v, (list, tuple)): list_of_v = maybe_list_of_v for v in list_of_v: # we're passing a short arg as a kwarg, example: # cut(d="\t") if len(k) == 1: if v is not False: processed.append("-" + k) if v is not True: processed.append(str(v)) # we're doing a long arg else: if not raw: k = k.replace("_", "-") # if it's true, it has no value, just pass the name if v is True: processed.append(prefix + k) # if it's false, skip passing it elif v is False: pass # we may need to break the argument up into multiple arguments elif sep is None or sep == " ": processed.append(prefix + k) processed.append(str(v)) # otherwise just join it together into a single argument else: arg = f"{prefix}{k}{sep}{v}" processed.append(arg) return processed def _start_daemon_thread(fn, name, exc_queue, *a): def wrap(*rgs, **kwargs): try: fn(*rgs, **kwargs) except Exception as e: exc_queue.put(e) raise thread = threading.Thread(target=wrap, name=name, args=a) thread.daemon = True thread.start() return thread def setwinsize(fd, rows_cols): """set the terminal size of a tty file descriptor. borrowed logic from pexpect.py""" rows, cols = rows_cols winsize = getattr(termios, "TIOCSWINSZ", -2146929561) s = struct.pack("HHHH", rows, cols, 0, 0) fcntl.ioctl(fd, winsize, s) def construct_streamreader_callback(process, handler): """here we're constructing a closure for our streamreader callback. this is used in the case that we pass a callback into _out or _err, meaning we want to our callback to handle each bit of output we construct the closure based on how many arguments it takes. the reason for this is to make it as easy as possible for people to use, without limiting them. a new user will assume the callback takes 1 argument (the data). as they get more advanced, they may want to terminate the process, or pass some stdin back, and will realize that they can pass a callback of more args""" # implied arg refers to the "self" that methods will pass in. we need to # account for this implied arg when figuring out what function the user # passed in based on number of args implied_arg = 0 partial_args = 0 handler_to_inspect = handler if isinstance(handler, partial): partial_args = len(handler.args) handler_to_inspect = handler.func if inspect.ismethod(handler_to_inspect): implied_arg = 1 num_args = get_num_args(handler_to_inspect) else: if inspect.isfunction(handler_to_inspect): num_args = get_num_args(handler_to_inspect) # is an object instance with __call__ method else: implied_arg = 1 num_args = get_num_args(handler_to_inspect.__call__) net_args = num_args - implied_arg - partial_args handler_args = () # just the chunk if net_args == 1: handler_args = () # chunk, stdin if net_args == 2: handler_args = (process.stdin,) # chunk, stdin, process elif net_args == 3: # notice we're only storing a weakref, to prevent cyclic references # (where the process holds a streamreader, and a streamreader holds a # handler-closure with a reference to the process handler_args = (process.stdin, weakref.ref(process)) def fn(chunk): # this is pretty ugly, but we're evaluating the process at call-time, # because it's a weakref a = handler_args if len(a) == 2: a = (handler_args[0], handler_args[1]()) return handler(chunk, *a) return fn def get_exc_exit_code_would_raise(exit_code, ok_codes, sigpipe_ok): exc = None success = exit_code in ok_codes signals_that_should_throw_exception = [ sig for sig in SIGNALS_THAT_SHOULD_THROW_EXCEPTION if -sig not in ok_codes ] bad_sig = -exit_code in signals_that_should_throw_exception # if this is a piped command, SIGPIPE must be ignored by us and not raise an # exception, since it's perfectly normal for the consumer of a process's # pipe to terminate early if sigpipe_ok and -exit_code == signal.SIGPIPE: bad_sig = False success = True if not success or bad_sig: exc = get_rc_exc(exit_code) return exc def handle_process_exit_code(exit_code): """this should only ever be called once for each child process""" # if we exited from a signal, let our exit code reflect that if os.WIFSIGNALED(exit_code): exit_code = -os.WTERMSIG(exit_code) # otherwise just give us a normal exit code elif os.WIFEXITED(exit_code): exit_code = os.WEXITSTATUS(exit_code) else: raise RuntimeError("Unknown child exit status!") return exit_code def no_interrupt(syscall, *args, **kwargs): """a helper for making system calls immune to EINTR""" ret = None while True: try: ret = syscall(*args, **kwargs) except OSError as e: if e.errno == errno.EINTR: continue else: raise else: break return ret class OProc: """this class is instantiated by RunningCommand for a command to be exec'd. it handles all the nasty business involved with correctly setting up the input/output to the child process. it gets its name for subprocess.Popen (process open) but we're calling ours OProc (open process)""" _default_window_size = (24, 80) # used in redirecting STDOUT = -1 STDERR = -2 def __init__( self, command, parent_log, cmd, stdin, stdout, stderr, call_args, pipe, process_assign_lock, ): """ cmd is the full list of arguments that will be exec'd. it includes the program name and all its arguments. stdin, stdout, stderr are what the child will use for standard input/output/err. call_args is a mapping of all the special keyword arguments to apply to the child process. """ self.command = command self.call_args = call_args # convenience ca = self.call_args if ca["uid"] is not None: if os.getuid() != 0: raise RuntimeError("UID setting requires root privileges") target_uid = ca["uid"] pwrec = pwd.getpwuid(ca["uid"]) target_gid = pwrec.pw_gid else: target_uid, target_gid = None, None # I had issues with getting 'Input/Output error reading stdin' from dd, # until I set _tty_out=False if ca["piped"]: ca["tty_out"] = False self._stdin_process = None # if the objects that we are passing to the OProc happen to be a # file-like object that is a tty, for example `sys.stdin`, then, later # on in this constructor, we're going to skip out on setting up pipes # and pseudoterminals for those endpoints stdin_is_fd_based = ob_is_fd_based(stdin) stdout_is_fd_based = ob_is_fd_based(stdout) stderr_is_fd_based = ob_is_fd_based(stderr) if isinstance(ca["tee"], (str, bool, int)) or ca["tee"] is None: tee = {ca["tee"]} else: tee = set(ca["tee"]) tee_out = TEE_STDOUT.intersection(tee) tee_err = TEE_STDERR.intersection(tee) single_tty = ca["tty_in"] and ca["tty_out"] and ca["unify_ttys"] # this logic is a little convoluted, but basically this top-level # if/else is for consolidating input and output TTYs into a single # TTY. this is the only way some secure programs like ssh will # output correctly (is if stdout and stdin are both the same TTY) if single_tty: # master_fd, slave_fd = pty.openpty() # # Anything that is written on the master end is provided to the process on # the slave end as though it was # input typed on a terminal. -"man 7 pty" # # later, in the child process, we're going to do this, so keep it in mind: # # os.dup2(self._stdin_child_fd, 0) # os.dup2(self._stdout_child_fd, 1) # os.dup2(self._stderr_child_fd, 2) self._stdin_parent_fd, self._stdin_child_fd = pty.openpty() # this makes our parent fds behave like a terminal. it says that the very # same fd that we "type" to (for stdin) is the same one that we see output # printed to (for stdout) self._stdout_parent_fd = os.dup(self._stdin_parent_fd) # this line is what makes stdout and stdin attached to the same pty. in # other words the process will write to the same underlying fd as stdout # as it uses to read from for stdin. this makes programs like ssh happy self._stdout_child_fd = os.dup(self._stdin_child_fd) self._stderr_parent_fd = os.dup(self._stdin_parent_fd) self._stderr_child_fd = os.dup(self._stdin_child_fd) # do not consolidate stdin and stdout. this is the most common use- # case else: # this check here is because we may be doing piping and so our stdin # might be an instance of OProc if isinstance(stdin, OProc) and stdin.call_args["piped"]: self._stdin_child_fd = stdin._pipe_fd self._stdin_parent_fd = None self._stdin_process = stdin elif stdin_is_fd_based: self._stdin_child_fd = os.dup(get_fileno(stdin)) self._stdin_parent_fd = None elif ca["tty_in"]: self._stdin_parent_fd, self._stdin_child_fd = pty.openpty() # tty_in=False is the default else: self._stdin_child_fd, self._stdin_parent_fd = os.pipe() if stdout_is_fd_based and not tee_out: self._stdout_child_fd = os.dup(get_fileno(stdout)) self._stdout_parent_fd = None # tty_out=True is the default elif ca["tty_out"]: self._stdout_parent_fd, self._stdout_child_fd = pty.openpty() else: self._stdout_parent_fd, self._stdout_child_fd = os.pipe() # unless STDERR is going to STDOUT, it ALWAYS needs to be a pipe, # and never a PTY. the reason for this is not totally clear to me, # but it has to do with the fact that if STDERR isn't set as the # CTTY (because STDOUT is), the STDERR buffer won't always flush # by the time the process exits, and the data will be lost. # i've only seen this on OSX. if stderr is OProc.STDOUT: # if stderr is going to stdout, but stdout is a tty or a pipe, # we should not specify a read_fd, because stdout is os.dup'ed # directly to the stdout fd (no pipe), and so stderr won't have # a slave end of a pipe either to dup if stdout_is_fd_based and not tee_out: self._stderr_parent_fd = None else: self._stderr_parent_fd = os.dup(self._stdout_parent_fd) self._stderr_child_fd = os.dup(self._stdout_child_fd) elif stderr_is_fd_based and not tee_err: self._stderr_child_fd = os.dup(get_fileno(stderr)) self._stderr_parent_fd = None else: self._stderr_parent_fd, self._stderr_child_fd = os.pipe() piped = ca["piped"] self._pipe_fd = None if piped: fd_to_use = self._stdout_parent_fd if piped == "err": fd_to_use = self._stderr_parent_fd self._pipe_fd = os.dup(fd_to_use) new_session = ca["new_session"] new_group = ca["new_group"] needs_ctty = ca["tty_in"] # if we need a controlling terminal, we have to be in a new session where we # are the session leader, otherwise we would need to take over the existing # process session, and we can't do that(?) if needs_ctty: new_session = True self.ctty = None if needs_ctty: self.ctty = os.ttyname(self._stdin_child_fd) gc_enabled = gc.isenabled() if gc_enabled: gc.disable() # for synchronizing session_pipe_read, session_pipe_write = os.pipe() exc_pipe_read, exc_pipe_write = os.pipe() # this pipe is for synchronizing with the child that the parent has # closed its in/out/err fds. this is a bug on OSX (but not linux), # where we can lose output sometimes, due to a race, if we do # os.close(self._stdout_child_fd) in the parent after the child starts # writing. if IS_MACOS: close_pipe_read, close_pipe_write = os.pipe() else: close_pipe_read, close_pipe_write = None, None # session id, group id, process id self.sid = None self.pgid = None self.pid = os.fork() # child if self.pid == 0: # pragma: no cover if IS_MACOS: os.read(close_pipe_read, 1) os.close(close_pipe_read) os.close(close_pipe_write) # this is critical # our exc_pipe_write must have CLOEXEC enabled. the reason for this is # tricky: if our child (the block we're in now), has an exception, we need # to be able to write to exc_pipe_write, so that when the parent does # os.read(exc_pipe_read), it gets our traceback. however, # os.read(exc_pipe_read) in the parent blocks, so if our child *doesn't* # have an exception, and doesn't close the writing end, it hangs forever. # not good! but obviously the child can't close the writing end until it # knows it's not going to have an exception, which is impossible to know # because but what if os.execv has an exception? so the answer is CLOEXEC, # so that the writing end of the pipe gets closed upon successful exec, # and the parent reading the read end won't block (close breaks the block). flags = fcntl.fcntl(exc_pipe_write, fcntl.F_GETFD) flags |= fcntl.FD_CLOEXEC fcntl.fcntl(exc_pipe_write, fcntl.F_SETFD, flags) try: # ignoring SIGHUP lets us persist even after the controlling terminal # is closed if ca["bg"] is True: signal.signal(signal.SIGHUP, signal.SIG_IGN) # python ignores SIGPIPE by default. we must make sure to put # this behavior back to the default for spawned processes, # otherwise SIGPIPE won't kill piped processes, which is what we # need, so that we can check the error code of the killed # process to see that SIGPIPE killed it signal.signal(signal.SIGPIPE, signal.SIG_DFL) # put our forked process in a new session? this will relinquish # any control of our inherited CTTY and also make our parent # process init if new_session: os.setsid() elif new_group: os.setpgid(0, 0) sid = os.getsid(0) pgid = os.getpgid(0) payload = (f"{sid},{pgid}").encode(DEFAULT_ENCODING) os.write(session_pipe_write, payload) if ca["tty_out"] and not stdout_is_fd_based and not single_tty: # set raw mode, so there isn't any weird translation of # newlines to \r\n and other oddities. we're not outputting # to a terminal anyways # # we HAVE to do this here, and not in the parent process, # because we have to guarantee that this is set before the # child process is run, and we can't do it twice. tty.setraw(self._stdout_child_fd) # if the parent-side fd for stdin exists, close it. the case # where it may not exist is if we're using piping if self._stdin_parent_fd: os.close(self._stdin_parent_fd) if self._stdout_parent_fd: os.close(self._stdout_parent_fd) if self._stderr_parent_fd: os.close(self._stderr_parent_fd) os.close(session_pipe_read) os.close(exc_pipe_read) cwd = ca["cwd"] if cwd: os.chdir(cwd) os.dup2(self._stdin_child_fd, 0) os.dup2(self._stdout_child_fd, 1) os.dup2(self._stderr_child_fd, 2) # set our controlling terminal, but only if we're using a tty # for stdin. it doesn't make sense to have a ctty otherwise if needs_ctty: tmp_fd = os.open(os.ttyname(0), os.O_RDWR) os.close(tmp_fd) if ca["tty_out"] and not stdout_is_fd_based: setwinsize(1, ca["tty_size"]) if ca["uid"] is not None: os.setgid(target_gid) os.setuid(target_uid) preexec_fn = ca["preexec_fn"] if callable(preexec_fn): preexec_fn() close_fds = ca["close_fds"] if ca["pass_fds"]: close_fds = True if close_fds: pass_fds = {0, 1, 2, exc_pipe_write} pass_fds.update(ca["pass_fds"]) # don't inherit file descriptors try: inherited_fds = os.listdir("/dev/fd") except OSError: # Some systems don't have /dev/fd. Raises OSError in # Python2, FileNotFoundError on Python3. The latter doesn't # exist on Python2, but inherits from IOError, which does. inherited_fds = os.listdir("/proc/self/fd") inherited_fds = {int(fd) for fd in inherited_fds} - pass_fds for fd in inherited_fds: try: os.close(fd) except OSError: pass # python=3.6, locale=c will fail test_unicode_arg if we don't # explicitly encode to bytes via our desired encoding. this does # not seem to be the case in other python versions, even if locale=c bytes_cmd = [c.encode(ca["encoding"]) for c in cmd] # actually execute the process if ca["env"] is None: os.execv(bytes_cmd[0], bytes_cmd) else: os.execve(bytes_cmd[0], bytes_cmd, ca["env"]) # we must ensure that we carefully exit the child process on # exception, otherwise the parent process code will be executed # twice on exception https://github.com/amoffat/sh/issues/202 # # if your parent process experiences an exit code 255, it is most # likely that an exception occurred between the fork of the child # and the exec. this should be reported. except Exception: # noqa: E722 # some helpful debugging tb = traceback.format_exc().encode("utf8", "ignore") try: os.write(exc_pipe_write, tb) except Exception as e: # dump to stderr if we cannot save it to exc_pipe_write sys.stderr.write(f"\nFATAL SH ERROR: {e}\n") finally: os._exit(255) # parent else: if gc_enabled: gc.enable() os.close(self._stdin_child_fd) os.close(self._stdout_child_fd) os.close(self._stderr_child_fd) # tell our child process that we've closed our write_fds, so it is # ok to proceed towards exec. see the comment where this pipe is # opened, for why this is necessary if IS_MACOS: os.close(close_pipe_read) os.write(close_pipe_write, str(1).encode(DEFAULT_ENCODING)) os.close(close_pipe_write) os.close(exc_pipe_write) fork_exc = os.read(exc_pipe_read, 1024**2) os.close(exc_pipe_read) if fork_exc: fork_exc = fork_exc.decode(DEFAULT_ENCODING) raise ForkException(fork_exc) os.close(session_pipe_write) sid, pgid = ( os.read(session_pipe_read, 1024).decode(DEFAULT_ENCODING).split(",") ) os.close(session_pipe_read) self.sid = int(sid) self.pgid = int(pgid) # used to determine what exception to raise. if our process was # killed via a timeout counter, we'll raise something different than # a SIGKILL exception self.timed_out = False self.started = time.time() self.cmd = cmd # exit code should only be manipulated from within self._wait_lock # to prevent race conditions self.exit_code = None self.stdin = stdin # this accounts for when _out is a callable that is passed stdin. in that # case, if stdin is unspecified, we must set it to a queue, so callbacks can # put things on it if callable(ca["out"]) and self.stdin is None: self.stdin = Queue() # _pipe_queue is used internally to hand off stdout from one process # to another. by default, all stdout from a process gets dumped # into this pipe queue, to be consumed in real time (hence the # thread-safe Queue), or at a potentially later time self._pipe_queue = Queue() # this is used to prevent a race condition when we're waiting for # a process to end, and the OProc's internal threads are also checking # for the processes's end self._wait_lock = threading.Lock() # these are for aggregating the stdout and stderr. we use a deque # because we don't want to overflow self._stdout = deque(maxlen=ca["internal_bufsize"]) self._stderr = deque(maxlen=ca["internal_bufsize"]) if ca["tty_in"] and not stdin_is_fd_based: setwinsize(self._stdin_parent_fd, ca["tty_size"]) self.log = parent_log.get_child("process", repr(self)) self.log.debug("started process") # disable echoing, but only if it's a tty that we created ourselves if ca["tty_in"] and not stdin_is_fd_based: attr = termios.tcgetattr(self._stdin_parent_fd) attr[3] &= ~termios.ECHO termios.tcsetattr(self._stdin_parent_fd, termios.TCSANOW, attr) # this represents the connection from a Queue object (or whatever # we're using to feed STDIN) to the process's STDIN fd self._stdin_stream = None if self._stdin_parent_fd: log = self.log.get_child("streamwriter", "stdin") self._stdin_stream = StreamWriter( log, self._stdin_parent_fd, self.stdin, ca["in_bufsize"], ca["encoding"], ca["tty_in"], ) stdout_pipe = None if pipe is OProc.STDOUT and not ca["no_pipe"]: stdout_pipe = self._pipe_queue # this represents the connection from a process's STDOUT fd to # wherever it has to go, sometimes a pipe Queue (that we will use # to pipe data to other processes), and also an internal deque # that we use to aggregate all the output save_stdout = not ca["no_out"] and (tee_out or stdout is None) pipe_out = ca["piped"] in ("out", True) pipe_err = ca["piped"] in ("err",) # if we're piping directly into another process's file descriptor, we # bypass reading from the stdout stream altogether, because we've # already hooked up this processes's stdout fd to the other # processes's stdin fd self._stdout_stream = None if not pipe_out and self._stdout_parent_fd: if callable(stdout): stdout = construct_streamreader_callback(self, stdout) self._stdout_stream = StreamReader( self.log.get_child("streamreader", "stdout"), self._stdout_parent_fd, stdout, self._stdout, ca["out_bufsize"], ca["encoding"], ca["decode_errors"], stdout_pipe, save_data=save_stdout, ) elif self._stdout_parent_fd: os.close(self._stdout_parent_fd) # if stderr is going to one place (because it's grouped with stdout, # or we're dealing with a single tty), then we don't actually need a # stream reader for stderr, because we've already set one up for # stdout above self._stderr_stream = None if ( stderr is not OProc.STDOUT and not single_tty and not pipe_err and self._stderr_parent_fd ): stderr_pipe = None if pipe is OProc.STDERR and not ca["no_pipe"]: stderr_pipe = self._pipe_queue save_stderr = not ca["no_err"] and (tee_err or stderr is None) if callable(stderr): stderr = construct_streamreader_callback(self, stderr) self._stderr_stream = StreamReader( Logger("streamreader"), self._stderr_parent_fd, stderr, self._stderr, ca["err_bufsize"], ca["encoding"], ca["decode_errors"], stderr_pipe, save_data=save_stderr, ) elif self._stderr_parent_fd: os.close(self._stderr_parent_fd) def timeout_fn(): self.timed_out = True try: self.signal(ca["timeout_signal"]) except ProcessLookupError: # Edge case: the process probably exited pass self._timeout_event = None self._timeout_timer = None if ca["timeout"]: self._timeout_event = threading.Event() self._timeout_timer = threading.Timer( ca["timeout"], self._timeout_event.set ) self._timeout_timer.start() # this is for cases where we know that the RunningCommand that was # launched was not .wait()ed on to complete. in those unique cases, # we allow the thread that processes output to report exceptions in # that thread. it's important that we only allow reporting of the # exception, and nothing else (like the additional stuff that # RunningCommand.wait() does), because we want the exception to be # re-raised in the future, if we DO call .wait() handle_exit_code = None if ( not self.command._spawned_and_waited and ca["bg_exc"] # we don't want background exceptions if we're doing async stuff, # because we want those to bubble up. and not ca["async"] ): def fn(exit_code): with process_assign_lock: return self.command.handle_command_exit_code(exit_code) handle_exit_code = fn self._quit_threads = threading.Event() thread_name = f"background thread for pid {self.pid}" self._bg_thread_exc_queue = Queue(1) self._background_thread = _start_daemon_thread( background_thread, thread_name, self._bg_thread_exc_queue, timeout_fn, self._timeout_event, handle_exit_code, self.is_alive, self._quit_threads, ) # start the main io threads. stdin thread is not needed if we are # connecting from another process's stdout pipe self._input_thread = None self._input_thread_exc_queue = Queue(1) if self._stdin_stream: close_before_term = not needs_ctty thread_name = f"STDIN thread for pid {self.pid}" self._input_thread = _start_daemon_thread( input_thread, thread_name, self._input_thread_exc_queue, self.log, self._stdin_stream, self.is_alive, self._quit_threads, close_before_term, ) # this event is for cases where the subprocess that we launch # launches its OWN subprocess and os.dup's the stdout/stderr fds to that # new subprocess. in that case, stdout and stderr will never EOF, # so our output_thread will never finish and will hang. this event # prevents that hanging self._stop_output_event = threading.Event() # we need to set up a callback to fire when our `output_thread` is about # to exit. this callback will set an asyncio Event, so that coroutiens can # be notified that our output is finished. # if the `sh` command was launched from within a thread (so we're not in # the main thread), then we won't have an event loop. try: loop = asyncio.get_running_loop() except RuntimeError: def output_complete(): pass else: def output_complete(): loop.call_soon_threadsafe(self.command.aio_output_complete.set) self._output_thread_exc_queue = Queue(1) thread_name = f"STDOUT/ERR thread for pid {self.pid}" self._output_thread = _start_daemon_thread( output_thread, thread_name, self._output_thread_exc_queue, self.log, self._stdout_stream, self._stderr_stream, self._timeout_event, self.is_alive, self._quit_threads, self._stop_output_event, output_complete, ) def __repr__(self): return f"" def change_in_bufsize(self, buf): self._stdin_stream.stream_bufferer.change_buffering(buf) def change_out_bufsize(self, buf): self._stdout_stream.stream_bufferer.change_buffering(buf) def change_err_bufsize(self, buf): self._stderr_stream.stream_bufferer.change_buffering(buf) @property def stdout(self): return "".encode(self.call_args["encoding"]).join(self._stdout) @property def stderr(self): return "".encode(self.call_args["encoding"]).join(self._stderr) def get_pgid(self): """return the CURRENT group id of the process. this differs from self.pgid in that this reflects the current state of the process, where self.pgid is the group id at launch""" return os.getpgid(self.pid) def get_sid(self): """return the CURRENT session id of the process. this differs from self.sid in that this reflects the current state of the process, where self.sid is the session id at launch""" return os.getsid(self.pid) def signal_group(self, sig): self.log.debug("sending signal %d to group", sig) os.killpg(self.get_pgid(), sig) def signal(self, sig): self.log.debug("sending signal %d", sig) os.kill(self.pid, sig) def kill_group(self): self.log.debug("killing group") self.signal_group(signal.SIGKILL) def kill(self): self.log.debug("killing") self.signal(signal.SIGKILL) def terminate(self): self.log.debug("terminating") self.signal(signal.SIGTERM) def is_alive(self): """polls if our child process has completed, without blocking. this method has side-effects, such as setting our exit_code, if we happen to see our child exit while this is running""" if self.exit_code is not None: return False, self.exit_code # what we're doing here essentially is making sure that the main thread # (or another thread), isn't calling .wait() on the process. because # .wait() calls os.waitpid(self.pid, 0), we can't do an os.waitpid # here...because if we did, and the process exited while in this # thread, the main thread's os.waitpid(self.pid, 0) would raise OSError # (because the process ended in another thread). # # so essentially what we're doing is, using this lock, checking if # we're calling .wait(), and if we are, let .wait() get the exit code # and handle the status, otherwise let us do it. # # Using a small timeout provides backpressure against code that spams # calls to .is_alive() which may block the main thread from acquiring # the lock otherwise. acquired = self._wait_lock.acquire(timeout=0.00001) if not acquired: if self.exit_code is not None: return False, self.exit_code return True, self.exit_code witnessed_end = False try: # WNOHANG is just that...we're calling waitpid without hanging... # essentially polling the process. the return result is (0, 0) if # there's no process status, so we check that pid == self.pid below # in order to determine how to proceed pid, exit_code = no_interrupt(os.waitpid, self.pid, os.WNOHANG) if pid == self.pid: self.exit_code = handle_process_exit_code(exit_code) witnessed_end = True return False, self.exit_code # no child process except OSError: return False, self.exit_code else: return True, self.exit_code finally: self._wait_lock.release() if witnessed_end: self._process_just_ended() def _process_just_ended(self): if self._timeout_timer: self._timeout_timer.cancel() done_callback = self.call_args["done"] if done_callback: success = self.exit_code in self.call_args["ok_code"] done_callback(success, self.exit_code) # this can only be closed at the end of the process, because it might be # the CTTY, and closing it prematurely will send a SIGHUP. we also # don't want to close it if there's a self._stdin_stream, because that # is in charge of closing it also if self._stdin_parent_fd and not self._stdin_stream: os.close(self._stdin_parent_fd) def wait(self): """waits for the process to complete, handles the exit code""" self.log.debug("acquiring wait lock to wait for completion") # using the lock in a with-context blocks, which is what we want if # we're running wait() with self._wait_lock: self.log.debug("got wait lock") witnessed_end = False if self.exit_code is None: self.log.debug("exit code not set, waiting on pid") pid, exit_code = no_interrupt(os.waitpid, self.pid, 0) # blocks self.exit_code = handle_process_exit_code(exit_code) witnessed_end = True else: self.log.debug( "exit code already set (%d), no need to wait", self.exit_code ) self._process_exit_cleanup(witnessed_end=witnessed_end) return self.exit_code def _process_exit_cleanup(self, witnessed_end): self._quit_threads.set() # we may not have a thread for stdin, if the pipe has been connected # via _piped="direct" if self._input_thread: self._input_thread.join() # wait, then signal to our output thread that the child process is # done, and we should have finished reading all the stdout/stderr # data that we can by now timer = threading.Timer(2.0, self._stop_output_event.set) timer.start() # wait for our stdout and stderr streamreaders to finish reading and # aggregating the process output self._output_thread.join() timer.cancel() self._background_thread.join() if witnessed_end: self._process_just_ended() def input_thread(log, stdin, is_alive, quit_thread, close_before_term): """this is run in a separate thread. it writes into our process's stdin (a streamwriter) and waits the process to end AND everything that can be written to be written""" closed = False alive = True poller = Poller() poller.register_write(stdin) while poller and alive: changed = poller.poll(1) for fd, events in changed: if events & (POLLER_EVENT_WRITE | POLLER_EVENT_HUP): log.debug("%r ready for more input", stdin) done = stdin.write() if done: poller.unregister(stdin) if close_before_term: stdin.close() closed = True alive, _ = is_alive() while alive: quit_thread.wait(1) alive, _ = is_alive() if not closed: stdin.close() def event_wait(ev, timeout=None): triggered = ev.wait(timeout) return triggered def background_thread( timeout_fn, timeout_event, handle_exit_code, is_alive, quit_thread ): """handles the timeout logic""" # if there's a timeout event, loop if timeout_event: while not quit_thread.is_set(): timed_out = event_wait(timeout_event, 0.1) if timed_out: timeout_fn() break # handle_exit_code will be a function ONLY if our command was NOT waited on # as part of its spawning. in other words, it's probably a background # command # # this reports the exit code exception in our thread. it's purely for the # user's awareness, and cannot be caught or used in any way, so it's ok to # suppress this during the tests if handle_exit_code and not RUNNING_TESTS: # pragma: no cover alive = True exit_code = None while alive: quit_thread.wait(1) alive, exit_code = is_alive() handle_exit_code(exit_code) def output_thread( log, stdout, stderr, timeout_event, is_alive, quit_thread, stop_output_event, output_complete, ): """this function is run in a separate thread. it reads from the process's stdout stream (a streamreader), and waits for it to claim that its done""" poller = Poller() if stdout is not None: poller.register_read(stdout) if stderr is not None: poller.register_read(stderr) # this is our poll loop for polling stdout or stderr that is ready to # be read and processed. if one of those streamreaders indicate that it # is done altogether being read from, we remove it from our list of # things to poll. when no more things are left to poll, we leave this # loop and clean up while poller: changed = no_interrupt(poller.poll, 0.1) for f, events in changed: if events & (POLLER_EVENT_READ | POLLER_EVENT_HUP): log.debug("%r ready to be read from", f) done = f.read() if done: poller.unregister(f) elif events & POLLER_EVENT_ERROR: # for some reason, we have to just ignore streams that have had an # error. i'm not exactly sure why, but don't remove this until we # figure that out, and create a test for it pass if timeout_event and timeout_event.is_set(): break if stop_output_event.is_set(): break # we need to wait until the process is guaranteed dead before closing our # outputs, otherwise SIGPIPE alive, _ = is_alive() while alive: quit_thread.wait(1) alive, _ = is_alive() if stdout: stdout.close() if stderr: stderr.close() output_complete() class DoneReadingForever(Exception): pass class NotYetReadyToRead(Exception): pass def determine_how_to_read_input(input_obj): """given some kind of input object, return a function that knows how to read chunks of that input object. each reader function should return a chunk and raise a DoneReadingForever exception, or return None, when there's no more data to read NOTE: the function returned does not need to care much about the requested buffering type (eg, unbuffered vs newline-buffered). the StreamBufferer will take care of that. these functions just need to return a reasonably-sized chunk of data.""" if isinstance(input_obj, Queue): log_msg = "queue" get_chunk = get_queue_chunk_reader(input_obj) elif callable(input_obj): log_msg = "callable" get_chunk = get_callable_chunk_reader(input_obj) # also handles stringio elif hasattr(input_obj, "read"): log_msg = "file descriptor" get_chunk = get_file_chunk_reader(input_obj) elif isinstance(input_obj, str): log_msg = "string" get_chunk = get_iter_string_reader(input_obj) elif isinstance(input_obj, bytes): log_msg = "bytes" get_chunk = get_iter_string_reader(input_obj) elif isinstance(input_obj, GeneratorType): log_msg = "generator" get_chunk = get_iter_chunk_reader(iter(input_obj)) elif input_obj is None: log_msg = "None" def raise_(): raise DoneReadingForever get_chunk = raise_ else: try: it = iter(input_obj) except TypeError: raise Exception("unknown input object") else: log_msg = "general iterable" get_chunk = get_iter_chunk_reader(it) return get_chunk, log_msg def get_queue_chunk_reader(stdin): def fn(): try: chunk = stdin.get(True, 0.1) except Empty: raise NotYetReadyToRead if chunk is None: raise DoneReadingForever return chunk return fn def get_callable_chunk_reader(stdin): def fn(): try: data = stdin() except DoneReadingForever: raise if not data: raise DoneReadingForever return data return fn def get_iter_string_reader(stdin): """return an iterator that returns a chunk of a string every time it is called. notice that even though bufsize_type might be line buffered, we're not doing any line buffering here. that's because our StreamBufferer handles all buffering. we just need to return a reasonable-sized chunk.""" bufsize = 1024 iter_str = (stdin[i : i + bufsize] for i in range(0, len(stdin), bufsize)) return get_iter_chunk_reader(iter_str) def get_iter_chunk_reader(stdin): def fn(): try: chunk = stdin.__next__() return chunk except StopIteration: raise DoneReadingForever return fn def get_file_chunk_reader(stdin): bufsize = 1024 def fn(): # python 3.* includes a fileno on stringios, but accessing it throws an # exception. that exception is how we'll know we can't do a poll on # stdin is_real_file = True try: stdin.fileno() except UnsupportedOperation: is_real_file = False # this poll is for files that may not yet be ready to read. we test # for fileno because StringIO/BytesIO cannot be used in a poll if is_real_file and hasattr(stdin, "fileno"): poller = Poller() poller.register_read(stdin) changed = poller.poll(0.1) ready = False for fd, events in changed: if events & (POLLER_EVENT_READ | POLLER_EVENT_HUP): ready = True if not ready: raise NotYetReadyToRead chunk = stdin.read(bufsize) if not chunk: raise DoneReadingForever else: return chunk return fn def bufsize_type_to_bufsize(bf_type): """for a given bufsize type, return the actual bufsize we will read. notice that although 1 means "newline-buffered", we're reading a chunk size of 1024. this is because we have to read something. we let a StreamBufferer instance handle splitting our chunk on newlines""" # newlines if bf_type == 1: bufsize = 1024 # unbuffered elif bf_type == 0: bufsize = 1 # or buffered by specific amount else: bufsize = bf_type return bufsize class StreamWriter: """StreamWriter reads from some input (the stdin param) and writes to a fd (the stream param). the stdin may be a Queue, a callable, something with the "read" method, a string, or an iterable""" def __init__(self, log, stream, stdin, bufsize_type, encoding, tty_in): self.stream = stream self.stdin = stdin self.log = log self.encoding = encoding self.tty_in = tty_in self.stream_bufferer = StreamBufferer(bufsize_type, self.encoding) self.get_chunk, log_msg = determine_how_to_read_input(stdin) self.log.debug("parsed stdin as a %s", log_msg) def fileno(self): """defining this allows us to do poll on an instance of this class""" return self.stream def write(self): """attempt to get a chunk of data to write to our child process's stdin, then write it. the return value answers the questions "are we done writing forever?" """ # get_chunk may sometimes return bytes, and sometimes return strings # because of the nature of the different types of STDIN objects we # support try: chunk = self.get_chunk() if chunk is None: raise DoneReadingForever except DoneReadingForever: self.log.debug("done reading") if self.tty_in: # EOF time try: char = termios.tcgetattr(self.stream)[6][termios.VEOF] except: # noqa: E722 char = chr(4).encode() # normally, one EOF should be enough to signal to an program # that is read()ing, to return 0 and be on your way. however, # some programs are misbehaved, like python3.1 and python3.2. # they don't stop reading sometimes after read() returns 0. # this can be demonstrated with the following program: # # import sys # sys.stdout.write(sys.stdin.read()) # # then type 'a' followed by ctrl-d 3 times. in python # 2.6,2.7,3.3,3.4,3.5,3.6, it only takes 2 ctrl-d to terminate. # however, in python 3.1 and 3.2, it takes all 3. # # so here we send an extra EOF along, just in case. i don't # believe it can hurt anything os.write(self.stream, char) os.write(self.stream, char) return True except NotYetReadyToRead: self.log.debug("received no data") return False # if we're not bytes, make us bytes if not isinstance(chunk, bytes): chunk = chunk.encode(self.encoding) for proc_chunk in self.stream_bufferer.process(chunk): self.log.debug("got chunk size %d: %r", len(proc_chunk), proc_chunk[:30]) self.log.debug("writing chunk to process") try: os.write(self.stream, proc_chunk) except OSError: self.log.debug("OSError writing stdin chunk") return True def close(self): self.log.debug("closing, but flushing first") chunk = self.stream_bufferer.flush() self.log.debug("got chunk size %d to flush: %r", len(chunk), chunk[:30]) try: if chunk: os.write(self.stream, chunk) except OSError: pass os.close(self.stream) def determine_how_to_feed_output(handler, encoding, decode_errors): if callable(handler): process, finish = get_callback_chunk_consumer(handler, encoding, decode_errors) # in py3, this is used for bytes elif isinstance(handler, BytesIO): process, finish = get_cstringio_chunk_consumer(handler) # in py3, this is used for unicode elif isinstance(handler, StringIO): process, finish = get_stringio_chunk_consumer(handler, encoding, decode_errors) elif hasattr(handler, "write"): process, finish = get_file_chunk_consumer(handler, decode_errors) else: try: handler = int(handler) except (ValueError, TypeError): def process(chunk): return False # noqa: E731 def finish(): return None # noqa: E731 else: process, finish = get_fd_chunk_consumer(handler, decode_errors) return process, finish def get_fd_chunk_consumer(handler, decode_errors): handler = fdopen(handler, "w", closefd=False) return get_file_chunk_consumer(handler, decode_errors) def get_file_chunk_consumer(handler, decode_errors): if getattr(handler, "encoding", None): def encode(chunk): return chunk.decode(handler.encoding, decode_errors) # noqa: E731 else: def encode(chunk): return chunk # noqa: E731 if hasattr(handler, "flush"): flush = handler.flush else: def flush(): return None # noqa: E731 def process(chunk): handler.write(encode(chunk)) # we should flush on an fd. chunk is already the correctly-buffered # size, so we don't need the fd buffering as well flush() return False def finish(): flush() return process, finish def get_callback_chunk_consumer(handler, encoding, decode_errors): def process(chunk): # try to use the encoding first, if that doesn't work, send # the bytes, because it might be binary try: chunk = chunk.decode(encoding, decode_errors) except UnicodeDecodeError: pass return handler(chunk) def finish(): pass return process, finish def get_cstringio_chunk_consumer(handler): def process(chunk): handler.write(chunk) return False def finish(): pass return process, finish def get_stringio_chunk_consumer(handler, encoding, decode_errors): def process(chunk): handler.write(chunk.decode(encoding, decode_errors)) return False def finish(): pass return process, finish class StreamReader: """reads from some output (the stream) and sends what it just read to the handler.""" def __init__( self, log, stream, handler, buffer, bufsize_type, encoding, decode_errors, pipe_queue=None, save_data=True, ): self.stream = stream self.buffer = buffer self.save_data = save_data self.encoding = encoding self.decode_errors = decode_errors self.pipe_queue = None if pipe_queue: self.pipe_queue = weakref.ref(pipe_queue) self.log = log self.stream_bufferer = StreamBufferer( bufsize_type, self.encoding, self.decode_errors ) self.bufsize = bufsize_type_to_bufsize(bufsize_type) self.process_chunk, self.finish_chunk_processor = determine_how_to_feed_output( handler, encoding, decode_errors ) self.should_quit = False def fileno(self): """defining this allows us to do poll on an instance of this class""" return self.stream def close(self): chunk = self.stream_bufferer.flush() self.log.debug("got chunk size %d to flush: %r", len(chunk), chunk[:30]) if chunk: self.write_chunk(chunk) self.finish_chunk_processor() if self.pipe_queue and self.save_data: self.pipe_queue().put(None) os.close(self.stream) def write_chunk(self, chunk): # in PY3, the chunk coming in will be bytes, so keep that in mind if not self.should_quit: self.should_quit = self.process_chunk(chunk) if self.save_data: self.buffer.append(chunk) if self.pipe_queue: self.log.debug("putting chunk onto pipe: %r", chunk[:30]) self.pipe_queue().put(chunk) def read(self): # if we're PY3, we're reading bytes, otherwise we're reading # str try: chunk = no_interrupt(os.read, self.stream, self.bufsize) except OSError as e: self.log.debug("got errno %d, done reading", e.errno) return True if not chunk: self.log.debug("got no chunk, done reading") return True self.log.debug("got chunk size %d: %r", len(chunk), chunk[:30]) for chunk in self.stream_bufferer.process(chunk): self.write_chunk(chunk) class StreamBufferer: """this is used for feeding in chunks of stdout/stderr, and breaking it up into chunks that will actually be put into the internal buffers. for example, if you have two processes, one being piped to the other, and you want that, first process to feed lines of data (instead of the chunks however they come in), OProc will use an instance of this class to chop up the data and feed it as lines to be sent down the pipe""" def __init__(self, buffer_type, encoding=DEFAULT_ENCODING, decode_errors="strict"): # 0 for unbuffered, 1 for line, everything else for that amount self.type = buffer_type self.buffer = [] self.n_buffer_count = 0 self.encoding = encoding self.decode_errors = decode_errors # this is for if we change buffering types. if we change from line # buffered to unbuffered, its very possible that our self.buffer list # has data that was being saved up (while we searched for a newline). # we need to use that up, so we don't lose it self._use_up_buffer_first = False # the buffering lock is used because we might change the buffering # types from a different thread. for example, if we have a stdout # callback, we might use it to change the way stdin buffers. so we # lock self._buffering_lock = threading.RLock() self.log = Logger("stream_bufferer") def change_buffering(self, new_type): # TODO, when we stop supporting 2.6, make this a with context self.log.debug("acquiring buffering lock for changing buffering") self._buffering_lock.acquire() self.log.debug("got buffering lock for changing buffering") try: if new_type == 0: self._use_up_buffer_first = True self.type = new_type finally: self._buffering_lock.release() self.log.debug("released buffering lock for changing buffering") def process(self, chunk): # MAKE SURE THAT THE INPUT IS PY3 BYTES # THE OUTPUT IS ALWAYS PY3 BYTES # TODO, when we stop supporting 2.6, make this a with context self.log.debug( "acquiring buffering lock to process chunk (buffering: %d)", self.type ) self._buffering_lock.acquire() self.log.debug("got buffering lock to process chunk (buffering: %d)", self.type) try: # unbuffered if self.type == 0: if self._use_up_buffer_first: self._use_up_buffer_first = False to_write = self.buffer self.buffer = [] to_write.append(chunk) return to_write return [chunk] # line buffered elif self.type == 1: total_to_write = [] nl = "\n".encode(self.encoding) while True: newline = chunk.find(nl) if newline == -1: break chunk_to_write = chunk[: newline + 1] if self.buffer: chunk_to_write = b"".join(self.buffer) + chunk_to_write self.buffer = [] self.n_buffer_count = 0 chunk = chunk[newline + 1 :] total_to_write.append(chunk_to_write) if chunk: self.buffer.append(chunk) self.n_buffer_count += len(chunk) return total_to_write # N size buffered else: total_to_write = [] while True: overage = self.n_buffer_count + len(chunk) - self.type if overage >= 0: ret = "".encode(self.encoding).join(self.buffer) + chunk chunk_to_write = ret[: self.type] chunk = ret[self.type :] total_to_write.append(chunk_to_write) self.buffer = [] self.n_buffer_count = 0 else: self.buffer.append(chunk) self.n_buffer_count += len(chunk) break return total_to_write finally: self._buffering_lock.release() self.log.debug( "released buffering lock for processing chunk (buffering: %d)", self.type, ) def flush(self): self.log.debug("acquiring buffering lock for flushing buffer") self._buffering_lock.acquire() self.log.debug("got buffering lock for flushing buffer") try: ret = "".encode(self.encoding).join(self.buffer) self.buffer = [] return ret finally: self._buffering_lock.release() self.log.debug("released buffering lock for flushing buffer") def with_lock(lock): def wrapped(fn): fn = contextmanager(fn) @contextmanager def wrapped2(*args, **kwargs): with lock: with fn(*args, **kwargs): yield return wrapped2 return wrapped @with_lock(PUSHD_LOCK) def pushd(path): """pushd changes the actual working directory for the duration of the context, unlike the _cwd arg this will work with other built-ins such as sh.glob correctly""" orig_path = os.getcwd() os.chdir(path) try: yield finally: os.chdir(orig_path) @contextmanager def _args(**kwargs): """allows us to temporarily override all the special keyword parameters in a with context""" kwargs_str = ",".join([f"{k}={v!r}" for k, v in kwargs.items()]) raise DeprecationWarning( f""" sh.args() has been deprecated because it was never thread safe. use the following instead: sh2 = sh({kwargs_str}) sh2.your_command() or sh2 = sh({kwargs_str}) from sh2 import your_command your_command() """ ) class Environment(dict): """this allows lookups to names that aren't found in the global scope to be searched for as a program name. for example, if "ls" isn't found in this module's scope, we consider it a system program and try to find it. we use a dict instead of just a regular object as the base class because the exec() statement used in the run_repl requires the "globals" argument to be a dictionary""" # this is a list of all of the names that the sh module exports that will # not resolve to functions. we don't want to accidentally shadow real # commands with functions/imports that we define in sh.py. for example, # "import time" may override the time system program allowlist = { "Command", "RunningCommand", "CommandNotFound", "DEFAULT_ENCODING", "DoneReadingForever", "ErrorReturnCode", "NotYetReadyToRead", "SignalException", "ForkException", "TimeoutException", "StreamBufferer", "_aggregate_keywords", "__project_url__", "__version__", "__file__", "_args", "pushd", "glob", "contrib", } def __init__(self, globs, baked_args=None): """baked_args are defaults for the 'sh' execution context. for example: tmp = sh(_out=StringIO()) 'out' would end up in here as an entry in the baked_args dict""" super(dict, self).__init__() self.globs = globs self.baked_args = baked_args or {} def __getitem__(self, k): if k == "args": # Let the deprecated '_args' context manager be imported as 'args' k = "_args" # if we're trying to import something real, see if it's in our global scope. # what defines "real" is that it's in our allowlist if k in self.allowlist: return self.globs[k] # somebody tried to be funny and do "from sh import *" if k == "__all__": warnings.warn( "Cannot import * from sh. Please import sh or import programs " "individually." ) return [] # check if we're naming a dynamically generated ReturnCode exception exc = get_exc_from_name(k) if exc: return exc # https://github.com/ipython/ipython/issues/2577 # https://github.com/amoffat/sh/issues/97#issuecomment-10610629 if k.startswith("__") and k.endswith("__"): raise AttributeError # is it a command? cmd = resolve_command(k, self.globs[Command.__name__], self.baked_args) if cmd: return cmd # is it a custom builtin? builtin = getattr(self, "b_" + k, None) if builtin: return builtin # how about an environment variable? # this check must come after testing if its a command, because on some # systems, there are an environment variables that can conflict with # command names. # https://github.com/amoffat/sh/issues/238 try: return os.environ[k] except KeyError: pass # nothing found, raise an exception raise CommandNotFound(k) # Methods that begin with "b_" are implementations of shell built-ins that # people are used to, but which may not have an executable equivalent. @staticmethod def b_which(program, paths=None): return _which(program, paths) class Contrib(ModuleType): # pragma: no cover @classmethod def __call__(cls, name): def wrapper1(fn): @property def cmd_getter(self): cmd = resolve_command(name, Command) if not cmd: raise CommandNotFound(name) new_cmd = fn(cmd) return new_cmd setattr(cls, name, cmd_getter) return fn return wrapper1 mod_name = __name__ + ".contrib" contrib = Contrib(mod_name) sys.modules[mod_name] = contrib @contrib("git") def git(orig): # pragma: no cover """most git commands play nicer without a TTY""" cmd = orig.bake(_tty_out=False) return cmd @contrib("bash") def bash(orig): cmd = orig.bake("-c") return cmd @contrib("sudo") def sudo(orig): # pragma: no cover """a nicer version of sudo that uses getpass to ask for a password, or allows the first argument to be a string password""" prompt = f"[sudo] password for {getpass.getuser()}: " def stdin(): pw = getpass.getpass(prompt=prompt) + "\n" yield pw def process(a, kwargs): password = kwargs.pop("password", None) if password is None: pass_getter = stdin() else: pass_getter = password.rstrip("\n") + "\n" kwargs["_in"] = pass_getter return a, kwargs cmd = orig.bake("-S", _arg_preprocess=process) return cmd @contrib("ssh") def ssh(orig): # pragma: no cover """An ssh command for automatic password login""" class SessionContent: def __init__(self): self.chars = deque(maxlen=50000) self.lines = deque(maxlen=5000) self.line_chars = [] self.last_line = "" self.cur_char = "" def append_char(self, char): if char == "\n": line = self.cur_line self.last_line = line self.lines.append(line) self.line_chars = [] else: self.line_chars.append(char) self.chars.append(char) self.cur_char = char @property def cur_line(self): line = "".join(self.line_chars) return line class SSHInteract: def __init__(self, prompt_match, pass_getter, out_handler, login_success): self.prompt_match = prompt_match self.pass_getter = pass_getter self.out_handler = out_handler self.login_success = login_success self.content = SessionContent() # some basic state self.pw_entered = False self.success = False def __call__(self, char, stdin): self.content.append_char(char) if self.pw_entered and not self.success: self.success = self.login_success(self.content) if self.success: return self.out_handler(self.content, stdin) if self.prompt_match(self.content): password = self.pass_getter() stdin.put(password + "\n") self.pw_entered = True def process(a, kwargs): real_out_handler = kwargs.pop("interact") password = kwargs.pop("password", None) login_success = kwargs.pop("login_success", None) prompt_match = kwargs.pop("prompt", None) prompt = "Please enter SSH password: " if prompt_match is None: def prompt_match(content): return content.cur_line.endswith("password: ") # noqa: E731 if password is None: def pass_getter(): return getpass.getpass(prompt=prompt) # noqa: E731 else: def pass_getter(): return password.rstrip("\n") # noqa: E731 if login_success is None: def login_success(content): return True # noqa: E731 kwargs["_out"] = SSHInteract( prompt_match, pass_getter, real_out_handler, login_success ) return a, kwargs cmd = orig.bake( _out_bufsize=0, _tty_in=True, _unify_ttys=True, _arg_preprocess=process ) return cmd def run_repl(env): # pragma: no cover print(f"\n>> sh v{__version__}\n>> https://github.com/amoffat/sh\n") while True: try: line = input("sh> ") except (ValueError, EOFError): break try: exec(compile(line, "", "single"), env, env) except SystemExit: break except: # noqa: E722 print(traceback.format_exc()) # cleans up our last line print("") # this is a thin wrapper around THIS module (we patch sys.modules[__name__]). # this is in the case that the user does a "from sh import whatever" # in other words, they only want to import certain programs, not the whole # system PATH worth of commands. in this case, we just proxy the # import lookup to our Environment class class SelfWrapper(ModuleType): def __init__(self, self_module, baked_args=None): # this is super ugly to have to copy attributes like this, # but it seems to be the only way to make reload() behave # nicely. if i make these attributes dynamic lookups in # __getattr__, reload sometimes chokes in weird ways... super().__init__( name=getattr(self_module, "__name__", None), doc=getattr(self_module, "__doc__", None), ) for attr in ["__builtins__", "__file__", "__package__"]: setattr(self, attr, getattr(self_module, attr, None)) # python 3.2 (2.7 and 3.3 work fine) breaks on osx (not ubuntu) # if we set this to None. and 3.3 needs a value for __path__ self.__path__ = [] self.__self_module = self_module # Copy the Command class and add any baked call kwargs to it command_cls = Command cls_attrs = command_cls.__dict__.copy() cls_attrs.pop("__dict__", None) if baked_args: call_args, _ = command_cls._extract_call_args(baked_args) cls_attrs["_call_args"] = cls_attrs["_call_args"].copy() cls_attrs["_call_args"].update(call_args) globs = globals().copy() globs[command_cls.__name__] = type( command_cls.__name__, command_cls.__bases__, cls_attrs ) self.__env = Environment(globs, baked_args=baked_args) def __getattr__(self, name): return self.__env[name] def bake(self, **kwargs): baked_args = self.__env.baked_args.copy() baked_args.update(kwargs) new_sh = self.__class__(self.__self_module, baked_args) return new_sh if __name__ == "__main__": # pragma: no cover # we're being run as a stand-alone script env = Environment(globals()) run_repl(env) else: # we're being imported from somewhere sys.modules[__name__] = SelfWrapper(sys.modules[__name__]) ================================================ FILE: tests/Dockerfile ================================================ FROM ubuntu:focal ARG cache_bust RUN apt update &&\ apt -y install locales RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 ENV TZ Etc/UTC ENV DEBIAN_FRONTEND noninteractive RUN apt -y install\ software-properties-common\ curl\ sudo\ lsof RUN add-apt-repository -y ppa:deadsnakes/ppa RUN apt update RUN apt -y install\ python3.8\ python3.9\ python3.10\ python3.11 RUN apt -y install\ python3.8-distutils\ python3.9-distutils\ && curl https://bootstrap.pypa.io/get-pip.py | python3.9 - ARG uid=1000 RUN groupadd -g $uid shtest\ && useradd -m -u $uid -g $uid shtest\ && gpasswd -a shtest sudo\ && echo "shtest:shtest" | chpasswd ENV TOX_PARALLEL_NO_SPINNER=1 USER shtest WORKDIR /home/shtest/ ENV PATH="/home/shtest/.local/bin:$PATH" RUN pip install tox flake8 black rstcheck mypy COPY README.rst sh.py .flake8 tox.ini tests/sh_test.py /home/shtest/ ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/sh_test.py ================================================ import asyncio import errno import fcntl import inspect import logging import os import platform import pty import resource import signal import stat import sys import tempfile import time import unittest import unittest.mock import warnings from asyncio.queues import Queue as AQueue from contextlib import contextmanager from functools import partial, wraps from hashlib import md5 from io import BytesIO, StringIO from os.path import dirname, exists, join, realpath, split from pathlib import Path import sh THIS_DIR = Path(__file__).resolve().parent RAND_BYTES = os.urandom(10) # we have to use the real path because on osx, /tmp is a symlink to # /private/tmp, and so assertions that gettempdir() == sh.pwd() will fail tempdir = Path(tempfile.gettempdir()).resolve() IS_MACOS = platform.system() in ("AIX", "Darwin") SIGNALS_THAT_SHOULD_THROW_EXCEPTION = [ signal.SIGABRT, signal.SIGBUS, signal.SIGFPE, signal.SIGILL, signal.SIGINT, signal.SIGKILL, signal.SIGPIPE, signal.SIGQUIT, signal.SIGSEGV, signal.SIGTERM, signal.SIGSYS, ] def hash(a: str): h = md5(a.encode("utf8") + RAND_BYTES) return h.hexdigest() def randomize_order(a, b): h1 = hash(a) h2 = hash(b) if h1 == h2: return 0 elif h1 < h2: return -1 else: return 1 unittest.TestLoader.sortTestMethodsUsing = staticmethod(randomize_order) # these 3 functions are helpers for modifying PYTHONPATH with a module's main # directory def append_pythonpath(env, path): key = "PYTHONPATH" pypath = [p for p in env.get(key, "").split(":") if p] pypath.insert(0, path) pypath = ":".join(pypath) env[key] = pypath def get_module_import_dir(m): mod_file = inspect.getsourcefile(m) is_package = mod_file.endswith("__init__.py") mod_dir = dirname(mod_file) if is_package: mod_dir, _ = split(mod_dir) return mod_dir def append_module_path(env, m): append_pythonpath(env, get_module_import_dir(m)) system_python = sh.Command(sys.executable) # this is to ensure that our `python` helper here is able to import our local sh # module, and not the system one baked_env = os.environ.copy() append_module_path(baked_env, sh) python = system_python.bake(_env=baked_env, _return_cmd=True) pythons = python.bake(_return_cmd=False) python_bg = system_python.bake(_env=baked_env, _bg=True) def requires_progs(*progs): missing = [] for prog in progs: try: sh.Command(prog) except sh.CommandNotFound: missing.append(prog) friendly_missing = ", ".join(missing) return unittest.skipUnless( len(missing) == 0, f"Missing required system programs: {friendly_missing}" ) requires_posix = unittest.skipUnless(os.name == "posix", "Requires POSIX") requires_utf8 = unittest.skipUnless( sh.DEFAULT_ENCODING == "UTF-8", "System encoding must be UTF-8" ) not_macos = unittest.skipUnless(not IS_MACOS, "Doesn't work on MacOS") def requires_poller(poller): use_select = bool(int(os.environ.get("SH_TESTS_USE_SELECT", "0"))) cur_poller = "select" if use_select else "poll" return unittest.skipUnless( cur_poller == poller, f"Only enabled for select.{cur_poller}" ) @contextmanager def ulimit(key, new_soft): soft, hard = resource.getrlimit(key) resource.setrlimit(key, (new_soft, hard)) try: yield finally: resource.setrlimit(key, (soft, hard)) def create_tmp_test(code, prefix="tmp", delete=True, **kwargs): """creates a temporary test file that lives on disk, on which we can run python with sh""" py = tempfile.NamedTemporaryFile(prefix=prefix, delete=delete) code = code.format(**kwargs) code = code.encode("UTF-8") py.write(code) py.flush() # make the file executable st = os.stat(py.name) os.chmod(py.name, st.st_mode | stat.S_IEXEC) # we don't explicitly close, because close will remove the file, and we # don't want that until the test case is done. so we let the gc close it # when it goes out of scope return py class BaseTests(unittest.TestCase): def setUp(self): warnings.simplefilter("ignore", ResourceWarning) def tearDown(self): warnings.simplefilter("default", ResourceWarning) def assert_oserror(self, num, fn, *args, **kwargs): try: fn(*args, **kwargs) except OSError as e: self.assertEqual(e.errno, num) def assert_deprecated(self, fn, *args, **kwargs): with warnings.catch_warnings(record=True) as w: fn(*args, **kwargs) self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, DeprecationWarning)) class ArgTests(BaseTests): def test_list_args(self): processed = sh._aggregate_keywords({"arg": [1, 2, 3]}, "=", "--") self.assertListEqual(processed, ["--arg=1", "--arg=2", "--arg=3"]) def test_bool_values(self): processed = sh._aggregate_keywords({"truthy": True, "falsey": False}, "=", "--") self.assertListEqual(processed, ["--truthy"]) def test_space_sep(self): processed = sh._aggregate_keywords({"arg": "123"}, " ", "--") self.assertListEqual(processed, ["--arg", "123"]) @requires_posix class FunctionalTests(BaseTests): def setUp(self): self._environ = os.environ.copy() super().setUp() def tearDown(self): os.environ = self._environ super().tearDown() def test_print_command(self): from sh import ls, which actual_location = which("ls").strip() out = str(ls) self.assertEqual(out, actual_location) def test_unicode_arg(self): from sh import echo test = "漢字" p = echo(test, _encoding="utf8") output = p.strip() self.assertEqual(test, output) def test_unicode_exception(self): from sh import ErrorReturnCode py = create_tmp_test("exit(1)") arg = "漢字" native_arg = arg try: python(py.name, arg, _encoding="utf8") except ErrorReturnCode as e: self.assertIn(native_arg, str(e)) else: self.fail("exception wasn't raised") def test_pipe_fd(self): py = create_tmp_test("""print("hi world")""") read_fd, write_fd = os.pipe() python(py.name, _out=write_fd) out = os.read(read_fd, 10) self.assertEqual(out, b"hi world\n") def test_trunc_exc(self): py = create_tmp_test( """ import sys sys.stdout.write("a" * 1000) sys.stderr.write("b" * 1000) exit(1) """ ) self.assertRaises(sh.ErrorReturnCode_1, python, py.name) def test_number_arg(self): py = create_tmp_test( """ from optparse import OptionParser parser = OptionParser() options, args = parser.parse_args() print(args[0]) """ ) out = python(py.name, 3).strip() self.assertEqual(out, "3") def test_arg_string_coercion(self): py = create_tmp_test( """ from argparse import ArgumentParser parser = ArgumentParser() parser.add_argument("-n", type=int) parser.add_argument("--number", type=int) ns = parser.parse_args() print(ns.n + ns.number) """ ) out = python(py.name, n=3, number=4, _long_sep=None).strip() self.assertEqual(out, "7") def test_empty_stdin_no_hang(self): py = create_tmp_test( """ import sys data = sys.stdin.read() sys.stdout.write("no hang") """ ) out = pythons(py.name, _in="", _timeout=2) self.assertEqual(out, "no hang") out = pythons(py.name, _in=None, _timeout=2) self.assertEqual(out, "no hang") def test_exit_code(self): from sh import ErrorReturnCode_3 py = create_tmp_test( """ exit(3) """ ) self.assertRaises(ErrorReturnCode_3, python, py.name) def test_patched_glob(self): from glob import glob py = create_tmp_test( """ import sys print(sys.argv[1:]) """ ) files = glob("*.faowjefoajweofj") out = python(py.name, files).strip() self.assertEqual(out, "['*.faowjefoajweofj']") def test_exit_code_with_hasattr(self): from sh import ErrorReturnCode_3 py = create_tmp_test( """ exit(3) """ ) try: out = python(py.name, _iter=True) # hasattr can swallow exceptions hasattr(out, "something_not_there") list(out) self.assertEqual(out.exit_code, 3) self.fail("Command exited with error, but no exception thrown") except ErrorReturnCode_3: pass def test_exit_code_from_exception(self): from sh import ErrorReturnCode_3 py = create_tmp_test( """ exit(3) """ ) self.assertRaises(ErrorReturnCode_3, python, py.name) try: python(py.name) except Exception as e: self.assertEqual(e.exit_code, 3) def test_stdin_from_string(self): from sh import sed self.assertEqual( sed(_in="one test three", e="s/test/two/").strip(), "one two three" ) def test_ok_code(self): from sh import ErrorReturnCode_1, ErrorReturnCode_2, ls exc_to_test = ErrorReturnCode_2 code_to_pass = 2 if IS_MACOS: exc_to_test = ErrorReturnCode_1 code_to_pass = 1 self.assertRaises(exc_to_test, ls, "/aofwje/garogjao4a/eoan3on") ls("/aofwje/garogjao4a/eoan3on", _ok_code=code_to_pass) ls("/aofwje/garogjao4a/eoan3on", _ok_code=[code_to_pass]) ls("/aofwje/garogjao4a/eoan3on", _ok_code=range(code_to_pass + 1)) def test_ok_code_none(self): py = create_tmp_test("exit(0)") python(py.name, _ok_code=None) def test_ok_code_exception(self): from sh import ErrorReturnCode_0 py = create_tmp_test("exit(0)") self.assertRaises(ErrorReturnCode_0, python, py.name, _ok_code=2) def test_none_arg(self): py = create_tmp_test( """ import sys print(sys.argv[1:]) """ ) maybe_arg = "some" out = python(py.name, maybe_arg).strip() self.assertEqual(out, "['some']") maybe_arg = None out = python(py.name, maybe_arg).strip() self.assertEqual(out, "[]") def test_quote_escaping(self): py = create_tmp_test( """ from optparse import OptionParser parser = OptionParser() options, args = parser.parse_args() print(args) """ ) out = python(py.name, "one two three").strip() self.assertEqual(out, "['one two three']") out = python(py.name, 'one "two three').strip() self.assertEqual(out, "['one \"two three']") out = python(py.name, "one", "two three").strip() self.assertEqual(out, "['one', 'two three']") out = python(py.name, "one", 'two "haha" three').strip() self.assertEqual(out, "['one', 'two \"haha\" three']") out = python(py.name, "one two's three").strip() self.assertEqual(out, '["one two\'s three"]') out = python(py.name, "one two's three").strip() self.assertEqual(out, '["one two\'s three"]') def test_multiple_pipes(self): import time py = create_tmp_test( """ import sys import os import time for l in "andrew": sys.stdout.write(l) time.sleep(.2) """ ) inc_py = create_tmp_test( """ import sys while True: letter = sys.stdin.read(1) if not letter: break sys.stdout.write(chr(ord(letter)+1)) """ ) def inc(*args, **kwargs): return python("-u", inc_py.name, *args, **kwargs) class Derp: def __init__(self): self.times = [] self.stdout = [] self.last_received = None def agg(self, line): self.stdout.append(line.strip()) now = time.time() if self.last_received: self.times.append(now - self.last_received) self.last_received = now derp = Derp() p = inc( _in=inc( _in=inc(_in=python("-u", py.name, _piped=True), _piped=True), _piped=True, ), _out=derp.agg, ) p.wait() self.assertEqual("".join(derp.stdout), "dqguhz") self.assertTrue(all([t > 0.15 for t in derp.times])) def test_manual_stdin_string(self): from sh import tr out = tr("[:lower:]", "[:upper:]", _in="andrew").strip() self.assertEqual(out, "ANDREW") def test_manual_stdin_iterable(self): from sh import tr test = ["testing\n", "herp\n", "derp\n"] out = tr("[:lower:]", "[:upper:]", _in=test) match = "".join([t.upper() for t in test]) self.assertEqual(out, match) def test_manual_stdin_file(self): import tempfile from sh import tr test_string = "testing\nherp\nderp\n" stdin = tempfile.NamedTemporaryFile() stdin.write(test_string.encode()) stdin.flush() stdin.seek(0) out = tr("[:lower:]", "[:upper:]", _in=stdin) self.assertEqual(out, test_string.upper()) def test_manual_stdin_queue(self): from sh import tr try: from Queue import Queue except ImportError: from queue import Queue test = ["testing\n", "herp\n", "derp\n"] q = Queue() for t in test: q.put(t) q.put(None) # EOF out = tr("[:lower:]", "[:upper:]", _in=q) match = "".join([t.upper() for t in test]) self.assertEqual(out, match) def test_environment(self): """tests that environments variables that we pass into sh commands exist in the environment, and on the sh module""" import os # this is the environment we'll pass into our commands env = {"HERP": "DERP"} # first we test that the environment exists in our child process as # we've set it py = create_tmp_test( """ import os for key in list(os.environ.keys()): if key != "HERP": del os.environ[key] print(dict(os.environ)) """ ) out = python(py.name, _env=env).strip() self.assertEqual(out, "{'HERP': 'DERP'}") py = create_tmp_test( """ import os, sys sys.path.insert(0, os.getcwd()) import sh for key in list(os.environ.keys()): if key != "HERP": del os.environ[key] print(dict(HERP=sh.HERP)) """ ) out = python(py.name, _env=env, _cwd=THIS_DIR).strip() self.assertEqual(out, "{'HERP': 'DERP'}") # Test that _env also accepts os.environ which is a mpping but not a dict. os.environ["HERP"] = "DERP" out = python(py.name, _env=os.environ, _cwd=THIS_DIR).strip() self.assertEqual(out, "{'HERP': 'DERP'}") def test_which(self): # Test 'which' as built-in function from sh import ls which = sh._SelfWrapper__env.b_which self.assertEqual(which("fjoawjefojawe"), None) self.assertEqual(which("ls"), str(ls)) def test_which_paths(self): # Test 'which' as built-in function which = sh._SelfWrapper__env.b_which py = create_tmp_test( """ print("hi") """ ) test_path = dirname(py.name) _, test_name = os.path.split(py.name) found_path = which(test_name) self.assertEqual(found_path, None) found_path = which(test_name, [test_path]) self.assertEqual(found_path, py.name) def test_no_close_fds(self): # guarantee some extra fds in our parent process that don't close on exec. we # have to explicitly do this because at some point (I believe python 3.4), # python started being more stringent with closing fds to prevent security # vulnerabilities. python 2.7, for example, doesn't set CLOEXEC on # tempfile.TemporaryFile()s # # https://www.python.org/dev/peps/pep-0446/ tmp = [tempfile.TemporaryFile() for i in range(10)] for t in tmp: flags = fcntl.fcntl(t.fileno(), fcntl.F_GETFD) flags &= ~fcntl.FD_CLOEXEC fcntl.fcntl(t.fileno(), fcntl.F_SETFD, flags) py = create_tmp_test( """ import os print(len(os.listdir("/dev/fd"))) """ ) out = python(py.name, _close_fds=False).strip() # pick some number greater than 4, since it's hard to know exactly how many fds # will be open/inherted in the child self.assertGreater(int(out), 7) for t in tmp: t.close() def test_close_fds(self): # guarantee some extra fds in our parent process that don't close on exec. # we have to explicitly do this because at some point (I believe python 3.4), # python started being more stringent with closing fds to prevent security # vulnerabilities. python 2.7, for example, doesn't set CLOEXEC on # tempfile.TemporaryFile()s # # https://www.python.org/dev/peps/pep-0446/ tmp = [tempfile.TemporaryFile() for i in range(10)] for t in tmp: flags = fcntl.fcntl(t.fileno(), fcntl.F_GETFD) flags &= ~fcntl.FD_CLOEXEC fcntl.fcntl(t.fileno(), fcntl.F_SETFD, flags) py = create_tmp_test( """ import os print(os.listdir("/dev/fd")) """ ) out = python(py.name).strip() self.assertEqual(out, "['0', '1', '2', '3']") for t in tmp: t.close() def test_pass_fds(self): # guarantee some extra fds in our parent process that don't close on exec. # we have to explicitly do this because at some point (I believe python 3.4), # python started being more stringent with closing fds to prevent security # vulnerabilities. python 2.7, for example, doesn't set CLOEXEC on # tempfile.TemporaryFile()s # # https://www.python.org/dev/peps/pep-0446/ tmp = [tempfile.TemporaryFile() for i in range(10)] for t in tmp: flags = fcntl.fcntl(t.fileno(), fcntl.F_GETFD) flags &= ~fcntl.FD_CLOEXEC fcntl.fcntl(t.fileno(), fcntl.F_SETFD, flags) last_fd = tmp[-1].fileno() py = create_tmp_test( """ import os print(os.listdir("/dev/fd")) """ ) out = python(py.name, _pass_fds=[last_fd]).strip() inherited = [0, 1, 2, 3, last_fd] inherited_str = [str(i) for i in inherited] self.assertEqual(out, str(inherited_str)) for t in tmp: t.close() def test_no_arg(self): import pwd from sh import whoami u1 = whoami().strip() u2 = pwd.getpwuid(os.geteuid())[0] self.assertEqual(u1, u2) def test_incompatible_special_args(self): from sh import ls self.assertRaises(TypeError, ls, _iter=True, _piped=True) def test_invalid_env(self): from sh import ls exc = TypeError self.assertRaises(exc, ls, _env="XXX") self.assertRaises(exc, ls, _env={"foo": 123}) self.assertRaises(exc, ls, _env={123: "bar"}) def test_exception(self): from sh import ErrorReturnCode_2 py = create_tmp_test( """ exit(2) """ ) self.assertRaises(ErrorReturnCode_2, python, py.name) def test_piped_exception1(self): from sh import ErrorReturnCode_2 py = create_tmp_test( """ import sys sys.stdout.write("line1\\n") sys.stdout.write("line2\\n") sys.stdout.flush() exit(2) """ ) py2 = create_tmp_test("") def fn(): list(python(python(py.name, _piped=True), "-u", py2.name, _iter=True)) self.assertRaises(ErrorReturnCode_2, fn) def test_piped_exception2(self): from sh import ErrorReturnCode_2 py = create_tmp_test( """ import sys sys.stdout.write("line1\\n") sys.stdout.write("line2\\n") sys.stdout.flush() exit(2) """ ) py2 = create_tmp_test("") def fn(): python(python(py.name, _piped=True), "-u", py2.name) self.assertRaises(ErrorReturnCode_2, fn) def test_command_not_found(self): from sh import CommandNotFound def do_import(): from sh import aowjgoawjoeijaowjellll # noqa: F401 self.assertRaises(ImportError, do_import) def do_import(): import sh sh.awoefaowejfw self.assertRaises(CommandNotFound, do_import) def do_import(): import sh sh.Command("ofajweofjawoe") self.assertRaises(CommandNotFound, do_import) def test_command_wrapper_equivalence(self): from sh import Command, ls, which self.assertEqual(Command(str(which("ls")).strip()), ls) def test_doesnt_execute_directories(self): save_path = os.environ["PATH"] bin_dir1 = tempfile.mkdtemp() bin_dir2 = tempfile.mkdtemp() gcc_dir1 = os.path.join(bin_dir1, "gcc") gcc_file2 = os.path.join(bin_dir2, "gcc") try: os.environ["PATH"] = os.pathsep.join((bin_dir1, bin_dir2)) # a folder named 'gcc', its executable, but should not be # discovered by internal which(1)-clone os.makedirs(gcc_dir1) # an executable named gcc -- only this should be executed bunk_header = "#!/bin/sh\necho $*" with open(gcc_file2, "w") as h: h.write(bunk_header) os.chmod(gcc_file2, int(0o755)) from sh import gcc self.assertEqual(gcc._path, gcc_file2) self.assertEqual( gcc("no-error", _return_cmd=True).stdout.strip(), b"no-error", ) finally: os.environ["PATH"] = save_path if exists(gcc_file2): os.unlink(gcc_file2) if exists(gcc_dir1): os.rmdir(gcc_dir1) if exists(bin_dir1): os.rmdir(bin_dir1) if exists(bin_dir1): os.rmdir(bin_dir2) def test_multiple_args_short_option(self): py = create_tmp_test( """ from optparse import OptionParser parser = OptionParser() parser.add_option("-l", dest="long_option") options, args = parser.parse_args() print(len(options.long_option.split())) """ ) num_args = int(python(py.name, l="one two three")) # noqa: E741 self.assertEqual(num_args, 3) num_args = int(python(py.name, "-l", "one's two's three's")) self.assertEqual(num_args, 3) def test_multiple_args_long_option(self): py = create_tmp_test( """ from optparse import OptionParser parser = OptionParser() parser.add_option("-l", "--long-option", dest="long_option") options, args = parser.parse_args() print(len(options.long_option.split())) """ ) num_args = int(python(py.name, long_option="one two three", nothing=False)) self.assertEqual(num_args, 3) num_args = int(python(py.name, "--long-option", "one's two's three's")) self.assertEqual(num_args, 3) num_args = int(python(py.name, long_option="one's two's three's")) self.assertEqual(num_args, 3) cmd = str(python.bake(py.name, long_option="one two three")) self.assertTrue(cmd.endswith(" '--long-option=one two three'"), cmd) def test_short_bool_option(self): py = create_tmp_test( """ from optparse import OptionParser parser = OptionParser() parser.add_option("-s", action="store_true", default=False, dest="short_option") options, args = parser.parse_args() print(options.short_option) """ ) self.assertTrue(python(py.name, s=True).strip() == "True") self.assertTrue(python(py.name, s=False).strip() == "False") self.assertTrue(python(py.name).strip() == "False") def test_long_bool_option(self): py = create_tmp_test( """ from optparse import OptionParser parser = OptionParser() parser.add_option("-l", "--long-option", action="store_true", default=False, \ dest="long_option") options, args = parser.parse_args() print(options.long_option) """ ) self.assertTrue(python(py.name, long_option=True).strip() == "True") self.assertTrue(python(py.name).strip() == "False") def test_false_bool_ignore(self): py = create_tmp_test( """ import sys print(sys.argv[1:]) """ ) test = True self.assertEqual(python(py.name, test and "-n").strip(), "['-n']") test = False self.assertEqual(python(py.name, test and "-n").strip(), "[]") def test_composition(self): py1 = create_tmp_test( """ import sys print(int(sys.argv[1]) * 2) """ ) py2 = create_tmp_test( """ import sys print(int(sys.argv[1]) + 1) """ ) res = python(py2.name, python(py1.name, 8)).strip() self.assertEqual("17", res) def test_incremental_composition(self): py1 = create_tmp_test( """ import sys print(int(sys.argv[1]) * 2) """ ) py2 = create_tmp_test( """ import sys print(int(sys.stdin.read()) + 1) """ ) res = python(py2.name, _in=python(py1.name, 8, _piped=True)).strip() self.assertEqual("17", res) def test_short_option(self): from sh import sh s1 = sh(c="echo test").strip() s2 = "test" self.assertEqual(s1, s2) def test_long_option(self): py = create_tmp_test( """ from optparse import OptionParser parser = OptionParser() parser.add_option("-l", "--long-option", action="store", default="", dest="long_option") options, args = parser.parse_args() print(options.long_option.upper()) """ ) self.assertTrue(python(py.name, long_option="testing").strip() == "TESTING") self.assertTrue(python(py.name).strip() == "") def test_raw_args(self): py = create_tmp_test( """ from optparse import OptionParser parser = OptionParser() parser.add_option("--long_option", action="store", default=None, dest="long_option1") parser.add_option("--long-option", action="store", default=None, dest="long_option2") options, args = parser.parse_args() if options.long_option1: print(options.long_option1.upper()) else: print(options.long_option2.upper()) """ ) self.assertEqual( python(py.name, {"long_option": "underscore"}).strip(), "UNDERSCORE" ) self.assertEqual(python(py.name, long_option="hyphen").strip(), "HYPHEN") def test_custom_separator(self): py = create_tmp_test( """ import sys print(sys.argv[1]) """ ) opt = {"long-option": "underscore"} correct = "--long-option=custom=underscore" out = python(py.name, opt, _long_sep="=custom=").strip() self.assertEqual(out, correct) # test baking too correct = "--long-option=baked=underscore" python_baked = python.bake(py.name, opt, _long_sep="=baked=") out = python_baked().strip() self.assertEqual(out, correct) def test_custom_separator_space(self): py = create_tmp_test( """ import sys print(str(sys.argv[1:])) """ ) opt = {"long-option": "space"} correct = ["--long-option", "space"] out = python(py.name, opt, _long_sep=" ").strip() self.assertEqual(out, str(correct)) def test_custom_long_prefix(self): py = create_tmp_test( """ import sys print(sys.argv[1]) """ ) out = python( py.name, {"long-option": "underscore"}, _long_prefix="-custom-" ).strip() self.assertEqual(out, "-custom-long-option=underscore") out = python(py.name, {"long-option": True}, _long_prefix="-custom-").strip() self.assertEqual(out, "-custom-long-option") # test baking too out = python.bake( py.name, {"long-option": "underscore"}, _long_prefix="-baked-" )().strip() self.assertEqual(out, "-baked-long-option=underscore") out = python.bake( py.name, {"long-option": True}, _long_prefix="-baked-" )().strip() self.assertEqual(out, "-baked-long-option") def test_command_wrapper(self): from sh import Command, which ls = Command(str(which("ls")).strip()) wc = Command(str(which("wc")).strip()) c1 = int(wc(l=True, _in=ls("-A1", THIS_DIR, _return_cmd=True))) # noqa: E741 c2 = len(os.listdir(THIS_DIR)) self.assertEqual(c1, c2) def test_background(self): import time from sh import sleep start = time.time() sleep_time = 0.5 p = sleep(sleep_time, _bg=True) now = time.time() self.assertLess(now - start, sleep_time) p.wait() now = time.time() self.assertGreater(now - start, sleep_time) def test_background_exception(self): py = create_tmp_test("exit(1)") p = python(py.name, _bg=True, _bg_exc=False) # should not raise self.assertRaises(sh.ErrorReturnCode_1, p.wait) # should raise def test_with_context(self): import getpass from sh import whoami py = create_tmp_test( """ import sys import os import subprocess print("with_context") subprocess.Popen(sys.argv[1:], shell=False).wait() """ ) cmd1 = python.bake(py.name, _with=True) with cmd1: out = whoami() self.assertIn("with_context", out) self.assertIn(getpass.getuser(), out) def test_with_context_args(self): import getpass from sh import whoami py = create_tmp_test( """ import sys import os import subprocess from optparse import OptionParser parser = OptionParser() parser.add_option("-o", "--opt", action="store_true", default=False, dest="opt") options, args = parser.parse_args() if options.opt: subprocess.Popen(args[0], shell=False).wait() """ ) with python(py.name, opt=True, _with=True): out = whoami() self.assertEqual(getpass.getuser(), out.strip()) with python(py.name, _with=True): out = whoami() self.assertEqual(out.strip(), "") def test_with_context_nested(self): echo_path = sh.echo._path with sh.echo.bake("test1", _with=True): with sh.echo.bake("test2", _with=True): out = sh.echo("test3") self.assertEqual(out.strip(), f"test1 {echo_path} test2 {echo_path} test3") def test_binary_input(self): py = create_tmp_test( """ import sys data = sys.stdin.read() sys.stdout.write(data) """ ) data = b"1234" out = pythons(py.name, _in=data) self.assertEqual(out, "1234") def test_err_to_out(self): py = create_tmp_test( """ import sys import os sys.stdout.write("stdout") sys.stdout.flush() sys.stderr.write("stderr") sys.stderr.flush() """ ) stdout = pythons(py.name, _err_to_out=True) self.assertEqual(stdout, "stdoutstderr") def test_err_to_out_and_sys_stdout(self): py = create_tmp_test( """ import sys import os sys.stdout.write("stdout") sys.stdout.flush() sys.stderr.write("stderr") sys.stderr.flush() """ ) master, slave = os.pipe() stdout = pythons(py.name, _err_to_out=True, _out=slave) self.assertEqual(stdout, "") self.assertEqual(os.read(master, 12), b"stdoutstderr") def test_err_piped(self): py = create_tmp_test( """ import sys sys.stderr.write("stderr") """ ) py2 = create_tmp_test( """ import sys while True: line = sys.stdin.read() if not line: break sys.stdout.write(line) """ ) out = pythons("-u", py2.name, _in=python("-u", py.name, _piped="err")) self.assertEqual(out, "stderr") def test_out_redirection(self): import tempfile py = create_tmp_test( """ import sys import os sys.stdout.write("stdout") sys.stderr.write("stderr") """ ) file_obj = tempfile.NamedTemporaryFile() out = python(py.name, _out=file_obj) self.assertEqual(len(out), 0) file_obj.seek(0) actual_out = file_obj.read() file_obj.close() self.assertNotEqual(len(actual_out), 0) # test with tee file_obj = tempfile.NamedTemporaryFile() out = python(py.name, _out=file_obj, _tee=True) self.assertGreater(len(out), 0) file_obj.seek(0) actual_out = file_obj.read() file_obj.close() self.assertGreater(len(actual_out), 0) def test_err_redirection(self): import tempfile py = create_tmp_test( """ import sys import os sys.stdout.write("stdout") sys.stderr.write("stderr") """ ) file_obj = tempfile.NamedTemporaryFile() p = python("-u", py.name, _err=file_obj) file_obj.seek(0) stderr = file_obj.read().decode() file_obj.close() self.assertEqual(p.stdout, b"stdout") self.assertEqual(stderr, "stderr") self.assertEqual(len(p.stderr), 0) # now with tee file_obj = tempfile.NamedTemporaryFile() p = python(py.name, _err=file_obj, _tee="err") file_obj.seek(0) stderr = file_obj.read().decode() file_obj.close() self.assertEqual(p.stdout, b"stdout") self.assertEqual(stderr, "stderr") self.assertGreater(len(p.stderr), 0) def test_out_and_err_redirection(self): import tempfile py = create_tmp_test( """ import sys import os sys.stdout.write("stdout") sys.stderr.write("stderr") """ ) err_file_obj = tempfile.NamedTemporaryFile() out_file_obj = tempfile.NamedTemporaryFile() p = python(py.name, _out=out_file_obj, _err=err_file_obj, _tee=("err", "out")) out_file_obj.seek(0) stdout = out_file_obj.read().decode() out_file_obj.close() err_file_obj.seek(0) stderr = err_file_obj.read().decode() err_file_obj.close() self.assertEqual(stdout, "stdout") self.assertEqual(p.stdout, b"stdout") self.assertEqual(stderr, "stderr") self.assertEqual(p.stderr, b"stderr") def test_tty_tee(self): py = create_tmp_test( """ import sys sys.stdout.write("stdout") """ ) read, write = pty.openpty() out = python("-u", py.name, _out=write).stdout tee = os.read(read, 6) self.assertEqual(out, b"") self.assertEqual(tee, b"stdout") os.close(write) os.close(read) read, write = pty.openpty() out = python("-u", py.name, _out=write, _tee=True).stdout tee = os.read(read, 6) self.assertEqual(out, b"stdout") self.assertEqual(tee, b"stdout") os.close(write) os.close(read) def test_err_redirection_actual_file(self): import tempfile file_obj = tempfile.NamedTemporaryFile() py = create_tmp_test( """ import sys import os sys.stdout.write("stdout") sys.stderr.write("stderr") """ ) stdout = pythons("-u", py.name, _err=file_obj.name) file_obj.seek(0) stderr = file_obj.read().decode() file_obj.close() self.assertEqual(stdout, "stdout") self.assertEqual(stderr, "stderr") def test_subcommand_and_bake(self): import getpass py = create_tmp_test( """ import sys import os import subprocess print("subcommand") subprocess.Popen(sys.argv[1:], shell=False).wait() """ ) cmd1 = python.bake(py.name) out = cmd1.whoami() self.assertIn("subcommand", out) self.assertIn(getpass.getuser(), out) def test_multiple_bakes(self): py = create_tmp_test( """ import sys sys.stdout.write(str(sys.argv[1:])) """ ) out = python.bake(py.name).bake("bake1").bake("bake2")() self.assertEqual("['bake1', 'bake2']", str(out)) def test_arg_preprocessor(self): py = create_tmp_test( """ import sys sys.stdout.write(str(sys.argv[1:])) """ ) def arg_preprocess(args, kwargs): args.insert(0, "preprocessed") kwargs["a-kwarg"] = 123 return args, kwargs cmd = pythons.bake(py.name, _arg_preprocess=arg_preprocess) out = cmd("arg") self.assertEqual("['preprocessed', 'arg', '--a-kwarg=123']", out) def test_bake_args_come_first(self): from sh import ls ls = ls.bake(h=True) ran = ls("-la", _return_cmd=True).ran ft = ran.index("-h") self.assertIn("-la", ran[ft:]) def test_output_equivalence(self): from sh import whoami iam1 = whoami() iam2 = whoami() self.assertEqual(iam1, iam2) # https://github.com/amoffat/sh/pull/252 def test_stdout_pipe(self): py = create_tmp_test( r""" import sys sys.stdout.write("foobar\n") """ ) read_fd, write_fd = os.pipe() python(py.name, _out=write_fd, u=True) def alarm(sig, action): self.fail("Timeout while reading from pipe") import signal signal.signal(signal.SIGALRM, alarm) signal.alarm(3) data = os.read(read_fd, 100) self.assertEqual(b"foobar\n", data) signal.alarm(0) signal.signal(signal.SIGALRM, signal.SIG_DFL) def test_stdout_callback(self): py = create_tmp_test( """ import sys import os for i in range(5): print(i) """ ) stdout = [] def agg(line): stdout.append(line) p = python("-u", py.name, _out=agg) p.wait() self.assertEqual(len(stdout), 5) def test_stdout_callback_no_wait(self): import time py = create_tmp_test( """ import sys import os import time for i in range(5): print(i) time.sleep(.5) """ ) stdout = [] def agg(line): stdout.append(line) python("-u", py.name, _out=agg, _bg=True) # we give a little pause to make sure that the NamedTemporaryFile # exists when the python process actually starts time.sleep(0.5) self.assertNotEqual(len(stdout), 5) def test_stdout_callback_line_buffered(self): py = create_tmp_test( """ import sys import os for i in range(5): print("herpderp") """ ) stdout = [] def agg(line): stdout.append(line) p = python("-u", py.name, _out=agg, _out_bufsize=1) p.wait() self.assertEqual(len(stdout), 5) def test_stdout_callback_line_unbuffered(self): py = create_tmp_test( """ import sys import os for i in range(5): print("herpderp") """ ) stdout = [] def agg(char): stdout.append(char) p = python("-u", py.name, _out=agg, _out_bufsize=0) p.wait() # + 5 newlines self.assertEqual(len(stdout), len("herpderp") * 5 + 5) def test_stdout_callback_buffered(self): py = create_tmp_test( """ import sys import os for i in range(5): sys.stdout.write("herpderp") """ ) stdout = [] def agg(chunk): stdout.append(chunk) p = python("-u", py.name, _out=agg, _out_bufsize=4) p.wait() self.assertEqual(len(stdout), len("herp") / 2 * 5) def test_stdout_callback_with_input(self): py = create_tmp_test( """ import sys import os for i in range(5): print(str(i)) derp = input("herp? ") print(derp) """ ) def agg(line, stdin): if line.strip() == "4": stdin.put("derp\n") p = python("-u", py.name, _out=agg, _tee=True) p.wait() self.assertIn("derp", p) def test_stdout_callback_exit(self): py = create_tmp_test( """ import sys import os for i in range(5): print(i) """ ) stdout = [] def agg(line): line = line.strip() stdout.append(line) if line == "2": return True p = python("-u", py.name, _out=agg, _tee=True) p.wait() self.assertIn("4", p) self.assertNotIn("4", stdout) def test_stdout_callback_terminate(self): import signal py = create_tmp_test( """ import sys import os import time for i in range(5): print(i) time.sleep(.5) """ ) stdout = [] def agg(line, stdin, process): line = line.strip() stdout.append(line) if line == "3": process.terminate() return True import sh caught_signal = False try: p = python("-u", py.name, _out=agg, _bg=True) p.wait() except sh.SignalException_SIGTERM: caught_signal = True self.assertTrue(caught_signal) self.assertEqual(p.process.exit_code, -signal.SIGTERM) self.assertNotIn("4", p) self.assertNotIn("4", stdout) def test_stdout_callback_kill(self): import signal py = create_tmp_test( """ import sys import os import time for i in range(5): print(i) time.sleep(.5) """ ) stdout = [] def agg(line, stdin, process): line = line.strip() stdout.append(line) if line == "3": process.kill() return True import sh caught_signal = False try: p = python("-u", py.name, _out=agg, _bg=True) p.wait() except sh.SignalException_SIGKILL: caught_signal = True self.assertTrue(caught_signal) self.assertEqual(p.process.exit_code, -signal.SIGKILL) self.assertNotIn("4", p) self.assertNotIn("4", stdout) def test_general_signal(self): from signal import SIGINT py = create_tmp_test( """ import sys import os import time import signal i = 0 def sig_handler(sig, frame): global i i = 42 signal.signal(signal.SIGINT, sig_handler) for _ in range(6): print(i) i += 1 sys.stdout.flush() time.sleep(2) """ ) stdout = [] def agg(line, stdin, process): line = line.strip() stdout.append(line) if line == "3": process.signal(SIGINT) return True p = python(py.name, _out=agg, _tee=True) p.wait() self.assertEqual(p.process.exit_code, 0) self.assertEqual(str(p), "0\n1\n2\n3\n42\n43\n") def test_iter_generator(self): py = create_tmp_test( """ import sys import os import time for i in range(42): print(i) sys.stdout.flush() """ ) out = [] for line in python(py.name, _iter=True): out.append(int(line.strip())) self.assertEqual(len(out), 42) self.assertEqual(sum(out), 861) def test_async(self): py = create_tmp_test( """ import os import time time.sleep(0.5) print("hello") """ ) alternating = [] async def producer(q): alternating.append(1) msg = await python(py.name, _async=True) alternating.append(1) await q.put(msg.strip()) async def consumer(q): await asyncio.sleep(0.1) alternating.append(2) msg = await q.get() self.assertEqual(msg, "hello") alternating.append(2) async def main(): q = AQueue() await asyncio.gather(producer(q), consumer(q)) asyncio.run(main()) self.assertListEqual(alternating, [1, 2, 1, 2]) def test_async_exc(self): py = create_tmp_test("""exit(34)""") async def producer(): await python(py.name, _async=True, _return_cmd=False) self.assertRaises(sh.ErrorReturnCode_34, asyncio.run, producer()) def test_async_iter(self): py = create_tmp_test( """ for i in range(5): print(i) """ ) # this list will prove that our coroutines are yielding to eachother as each # line is produced alternating = [] async def producer(q): async for line in python(py.name, _iter=True): alternating.append(1) await q.put(int(line.strip())) await q.put(None) async def consumer(q): while True: line = await q.get() if line is None: return alternating.append(2) async def main(): q = AQueue() await asyncio.gather(producer(q), consumer(q)) asyncio.run(main()) self.assertListEqual(alternating, [1, 2, 1, 2, 1, 2, 1, 2, 1, 2]) def test_async_iter_exc(self): py = create_tmp_test( """ for i in range(5): print(i) exit(34) """ ) lines = [] async def producer(): async for line in python(py.name, _async=True): lines.append(int(line.strip())) self.assertRaises(sh.ErrorReturnCode_34, asyncio.run, producer()) def test_async_return_cmd(self): py = create_tmp_test( """ import sys sys.exit(0) """ ) async def main(): result = await python(py.name, _async=True, _return_cmd=True) self.assertIsInstance(result, sh.RunningCommand) result_str = await python(py.name, _async=True, _return_cmd=False) self.assertIsInstance(result_str, str) asyncio.run(main()) def test_async_return_cmd_exc(self): py = create_tmp_test( """ import sys sys.exit(1) """ ) async def main(): await python(py.name, _async=True, _return_cmd=True) self.assertRaises(sh.ErrorReturnCode_1, asyncio.run, main()) def test_handle_both_out_and_err(self): py = create_tmp_test( """ import sys import os import time for i in range(42): sys.stdout.write(str(i) + "\\n") sys.stdout.flush() if i % 2 == 0: sys.stderr.write(str(i) + "\\n") sys.stderr.flush() """ ) out = [] def handle_out(line): out.append(int(line.strip())) err = [] def handle_err(line): err.append(int(line.strip())) p = python(py.name, _err=handle_err, _out=handle_out, _bg=True) p.wait() self.assertEqual(sum(out), 861) self.assertEqual(sum(err), 420) def test_iter_unicode(self): # issue https://github.com/amoffat/sh/issues/224 test_string = "\xe4\xbd\x95\xe4\xbd\x95\n" * 150 # len > buffer_s txt = create_tmp_test(test_string) for line in sh.cat(txt.name, _iter=True): break self.assertLess(len(line), 1024) def test_nonblocking_iter(self): from errno import EWOULDBLOCK py = create_tmp_test( """ import time import sys time.sleep(1) sys.stdout.write("stdout") """ ) count = 0 value = None for line in python(py.name, _iter_noblock=True): if line == EWOULDBLOCK: count += 1 else: value = line self.assertGreater(count, 0) self.assertEqual(value, "stdout") py = create_tmp_test( """ import time import sys time.sleep(1) sys.stderr.write("stderr") """ ) count = 0 value = None for line in python(py.name, _iter_noblock="err"): if line == EWOULDBLOCK: count += 1 else: value = line self.assertGreater(count, 0) self.assertEqual(value, "stderr") def test_for_generator_to_err(self): py = create_tmp_test( """ import sys import os for i in range(42): sys.stderr.write(str(i)+"\\n") """ ) out = [] for line in python("-u", py.name, _iter="err"): out.append(line) self.assertEqual(len(out), 42) # verify that nothing is going to stdout out = [] for line in python("-u", py.name, _iter="out"): out.append(line) self.assertEqual(len(out), 0) def test_sigpipe(self): py1 = create_tmp_test( """ import sys import os import time import signal # by default, python disables SIGPIPE, in favor of using IOError exceptions, so # let's put that back to the system default where we terminate with a signal # exit code signal.signal(signal.SIGPIPE, signal.SIG_DFL) for letter in "andrew": time.sleep(0.6) print(letter) """ ) py2 = create_tmp_test( """ import sys import os import time while True: line = sys.stdin.readline() if not line: break print(line.strip().upper()) exit(0) """ ) p1 = python("-u", py1.name, _piped="out") p2 = python( "-u", py2.name, _in=p1, ) # SIGPIPE should happen, but it shouldn't be an error, since _piped is # truthful self.assertEqual(-p1.exit_code, signal.SIGPIPE) self.assertEqual(p2.exit_code, 0) def test_piped_generator(self): import time py1 = create_tmp_test( """ import sys import os import time for letter in "andrew": time.sleep(0.6) print(letter) """ ) py2 = create_tmp_test( """ import sys import os import time while True: line = sys.stdin.readline() if not line: break print(line.strip().upper()) """ ) times = [] last_received = None letters = "" for line in python( "-u", py2.name, _iter=True, _in=python("-u", py1.name, _piped="out") ): letters += line.strip() now = time.time() if last_received: times.append(now - last_received) last_received = now self.assertEqual("ANDREW", letters) self.assertTrue(all([t > 0.3 for t in times])) def test_no_out_iter_err(self): py = create_tmp_test( """ import sys sys.stderr.write("1\\n") sys.stderr.write("2\\n") sys.stderr.write("3\\n") sys.stderr.flush() """ ) nums = [int(num.strip()) for num in python(py.name, _iter="err", _no_out=True)] assert nums == [1, 2, 3] def test_generator_and_callback(self): py = create_tmp_test( """ import sys import os for i in range(42): sys.stderr.write(str(i * 2)+"\\n") print(i) """ ) stderr = [] def agg(line): stderr.append(int(line.strip())) out = [] for line in python("-u", py.name, _iter=True, _err=agg): out.append(line) self.assertEqual(len(out), 42) self.assertEqual(sum(stderr), 1722) def test_cast_bg(self): py = create_tmp_test( """ import sys import time time.sleep(0.5) sys.stdout.write(sys.argv[1]) """ ) self.assertEqual(int(python(py.name, "123", _bg=True)), 123) self.assertEqual(float(python(py.name, "789", _bg=True)), 789.0) def test_cmd_eq(self): py = create_tmp_test("") cmd1 = python.bake(py.name, "-u") cmd2 = python.bake(py.name, "-u") cmd3 = python.bake(py.name) self.assertEqual(cmd1, cmd2) self.assertNotEqual(cmd1, cmd3) def test_fg(self): py = create_tmp_test("exit(0)") # notice we're using `system_python`, and not `python`. this is because # `python` has an env baked into it, and we want `_env` to be None for # coverage system_python(py.name, _fg=True) def test_fg_false(self): """https://github.com/amoffat/sh/issues/520""" py = create_tmp_test("print('hello')") buf = StringIO() python(py.name, _fg=False, _out=buf) self.assertEqual(buf.getvalue(), "hello\n") def test_fg_true(self): """https://github.com/amoffat/sh/issues/520""" py = create_tmp_test("print('hello')") buf = StringIO() self.assertRaises(TypeError, python, py.name, _fg=True, _out=buf) def test_fg_env(self): py = create_tmp_test( """ import os code = int(os.environ.get("EXIT", "0")) exit(code) """ ) env = os.environ.copy() env["EXIT"] = "3" self.assertRaises(sh.ErrorReturnCode_3, python, py.name, _fg=True, _env=env) def test_fg_alternative(self): py = create_tmp_test("exit(0)") python(py.name, _in=sys.stdin, _out=sys.stdout, _err=sys.stderr) def test_fg_exc(self): py = create_tmp_test("exit(1)") self.assertRaises(sh.ErrorReturnCode_1, python, py.name, _fg=True) def test_out_filename(self): outfile = tempfile.NamedTemporaryFile() py = create_tmp_test("print('output')") python(py.name, _out=outfile.name) outfile.seek(0) self.assertEqual(b"output\n", outfile.read()) def test_out_pathlike(self): from pathlib import Path outfile = tempfile.NamedTemporaryFile() py = create_tmp_test("print('output')") python(py.name, _out=Path(outfile.name)) outfile.seek(0) self.assertEqual(b"output\n", outfile.read()) def test_bg_exit_code(self): py = create_tmp_test( """ import time time.sleep(1) exit(49) """ ) p = python(py.name, _ok_code=49, _bg=True) self.assertEqual(49, p.exit_code) def test_cwd(self): from os.path import realpath from sh import pwd self.assertEqual(str(pwd(_cwd="/tmp")), realpath("/tmp") + "\n") self.assertEqual(str(pwd(_cwd="/etc")), realpath("/etc") + "\n") def test_cwd_fg(self): td = realpath(tempfile.mkdtemp()) py = create_tmp_test( f""" import sh import os from os.path import realpath orig = realpath(os.getcwd()) print(orig) sh.pwd(_cwd="{td}", _fg=True) print(realpath(os.getcwd())) """ ) orig, newdir, restored = python(py.name).strip().split("\n") newdir = realpath(newdir) self.assertEqual(newdir, td) self.assertEqual(orig, restored) self.assertNotEqual(orig, newdir) os.rmdir(td) def test_huge_piped_data(self): from sh import tr stdin = tempfile.NamedTemporaryFile() data = "herpderp" * 4000 + "\n" stdin.write(data.encode()) stdin.flush() stdin.seek(0) out = tr("[:upper:]", "[:lower:]", _in=tr("[:lower:]", "[:upper:]", _in=data)) self.assertTrue(out == data) def test_tty_input(self): py = create_tmp_test( """ import sys import os if os.isatty(sys.stdin.fileno()): sys.stdout.write("password?\\n") sys.stdout.flush() pw = sys.stdin.readline().strip() sys.stdout.write("%s\\n" % ("*" * len(pw))) sys.stdout.flush() else: sys.stdout.write("no tty attached!\\n") sys.stdout.flush() """ ) test_pw = "test123" expected_stars = "*" * len(test_pw) d = {} def password_enterer(line, stdin): line = line.strip() if not line: return if line == "password?": stdin.put(test_pw + "\n") elif line.startswith("*"): d["stars"] = line return True pw_stars = python(py.name, _tty_in=True, _out=password_enterer) pw_stars.wait() self.assertEqual(d["stars"], expected_stars) response = python(py.name) self.assertEqual(str(response), "no tty attached!\n") def test_tty_output(self): py = create_tmp_test( """ import sys import os if os.isatty(sys.stdout.fileno()): sys.stdout.write("tty attached") sys.stdout.flush() else: sys.stdout.write("no tty attached") sys.stdout.flush() """ ) out = pythons(py.name, _tty_out=True) self.assertEqual(out, "tty attached") out = pythons(py.name, _tty_out=False) self.assertEqual(out, "no tty attached") def test_stringio_output(self): import sh py = create_tmp_test( """ import sys sys.stdout.write(sys.argv[1]) """ ) out = StringIO() sh.python(py.name, "testing 123", _out=out) self.assertEqual(out.getvalue(), "testing 123") out = BytesIO() sh.python(py.name, "testing 123", _out=out) self.assertEqual(out.getvalue().decode(), "testing 123") def test_stringio_input(self): from sh import cat input = StringIO() input.write("herpderp") input.seek(0) out = cat(_in=input) self.assertEqual(out, "herpderp") def test_internal_bufsize(self): from sh import cat output = cat(_in="a" * 1000, _internal_bufsize=100, _out_bufsize=0) self.assertEqual(len(output), 100) output = cat(_in="a" * 1000, _internal_bufsize=50, _out_bufsize=2) self.assertEqual(len(output), 100) def test_change_stdout_buffering(self): py = create_tmp_test( """ import sys import os # this proves that we won't get the output into our callback until we send # a newline sys.stdout.write("switch ") sys.stdout.flush() sys.stdout.write("buffering\\n") sys.stdout.flush() sys.stdin.read(1) sys.stdout.write("unbuffered") sys.stdout.flush() # this is to keep the output from being flushed by the process ending, which # would ruin our test. we want to make sure we get the string "unbuffered" # before the process ends, without writing a newline sys.stdin.read(1) """ ) d = { "newline_buffer_success": False, "unbuffered_success": False, } def interact(line, stdin, process): line = line.strip() if not line: return if line == "switch buffering": d["newline_buffer_success"] = True process.change_out_bufsize(0) stdin.put("a") elif line == "unbuffered": stdin.put("b") d["unbuffered_success"] = True return True # start with line buffered stdout pw_stars = python("-u", py.name, _out=interact, _out_bufsize=1) pw_stars.wait() self.assertTrue(d["newline_buffer_success"]) self.assertTrue(d["unbuffered_success"]) def test_callable_interact(self): py = create_tmp_test( """ import sys sys.stdout.write("line1") """ ) class Callable: def __init__(self): self.line = None def __call__(self, line): self.line = line cb = Callable() python(py.name, _out=cb) self.assertEqual(cb.line, "line1") def test_encoding(self): return self.skipTest( "what's the best way to test a different '_encoding' special keyword" "argument?" ) def test_timeout(self): from time import time import sh sleep_for = 3 timeout = 1 started = time() try: sh.sleep(sleep_for, _timeout=timeout).wait() except sh.TimeoutException as e: assert "sleep 3" in e.full_cmd else: self.fail("no timeout exception") elapsed = time() - started self.assertLess(abs(elapsed - timeout), 0.5) def test_timeout_overstep(self): started = time.time() sh.sleep(1, _timeout=5) elapsed = time.time() - started self.assertLess(abs(elapsed - 1), 0.5) def test_timeout_wait(self): p = sh.sleep(3, _bg=True) self.assertRaises(sh.TimeoutException, p.wait, timeout=1) def test_timeout_wait_overstep(self): p = sh.sleep(1, _bg=True) p.wait(timeout=5) def test_timeout_wait_negative(self): p = sh.sleep(3, _bg=True) self.assertRaises(RuntimeError, p.wait, timeout=-3) def test_binary_pipe(self): binary = b"\xec;\xedr\xdbF" py1 = create_tmp_test( """ import sys import os sys.stdout = os.fdopen(sys.stdout.fileno(), "wb", 0) sys.stdout.write(b'\\xec;\\xedr\\xdbF') """ ) py2 = create_tmp_test( """ import sys import os sys.stdin = os.fdopen(sys.stdin.fileno(), "rb", 0) sys.stdout = os.fdopen(sys.stdout.fileno(), "wb", 0) sys.stdout.write(sys.stdin.read()) """ ) out = python(py2.name, _in=python(py1.name)) self.assertEqual(out.stdout, binary) # designed to trigger the "... (%d more, please see e.stdout)" output # of the ErrorReturnCode class def test_failure_with_large_output(self): from sh import ErrorReturnCode_1 py = create_tmp_test( """ print("andrewmoffat" * 1000) exit(1) """ ) self.assertRaises(ErrorReturnCode_1, python, py.name) # designed to check if the ErrorReturnCode constructor does not raise # an UnicodeDecodeError def test_non_ascii_error(self): from sh import ErrorReturnCode, ls test = "/á" self.assertRaises(ErrorReturnCode, ls, test, _encoding="utf8") def test_no_out(self): py = create_tmp_test( """ import sys sys.stdout.write("stdout") sys.stderr.write("stderr") """ ) p = python(py.name, _no_out=True) self.assertEqual(p.stdout, b"") self.assertEqual(p.stderr, b"stderr") self.assertTrue(p.process._pipe_queue.empty()) def callback(line): pass p = python(py.name, _out=callback) self.assertEqual(p.stdout, b"") self.assertEqual(p.stderr, b"stderr") self.assertTrue(p.process._pipe_queue.empty()) p = python(py.name) self.assertEqual(p.stdout, b"stdout") self.assertEqual(p.stderr, b"stderr") self.assertFalse(p.process._pipe_queue.empty()) def test_tty_stdin(self): py = create_tmp_test( """ import sys sys.stdout.write(sys.stdin.read()) sys.stdout.flush() """ ) out = pythons(py.name, _in="test\n", _tty_in=True) self.assertEqual("test\n", out) def test_no_err(self): py = create_tmp_test( """ import sys sys.stdout.write("stdout") sys.stderr.write("stderr") """ ) p = python(py.name, _no_err=True) self.assertEqual(p.stderr, b"") self.assertEqual(p.stdout, b"stdout") self.assertFalse(p.process._pipe_queue.empty()) def callback(line): pass p = python(py.name, _err=callback) self.assertEqual(p.stderr, b"") self.assertEqual(p.stdout, b"stdout") self.assertFalse(p.process._pipe_queue.empty()) p = python(py.name) self.assertEqual(p.stderr, b"stderr") self.assertEqual(p.stdout, b"stdout") self.assertFalse(p.process._pipe_queue.empty()) def test_no_pipe(self): from sh import ls # calling a command regular should fill up the pipe_queue p = ls(_return_cmd=True) self.assertFalse(p.process._pipe_queue.empty()) # calling a command with a callback should not def callback(line): pass p = ls(_out=callback, _return_cmd=True) self.assertTrue(p.process._pipe_queue.empty()) # calling a command regular with no_pipe also should not p = ls(_no_pipe=True, _return_cmd=True) self.assertTrue(p.process._pipe_queue.empty()) def test_decode_error_handling(self): from functools import partial py = create_tmp_test( """ # -*- coding: utf8 -*- import sys import os sys.stdout = os.fdopen(sys.stdout.fileno(), 'wb') sys.stdout.write(bytes("te漢字st", "utf8") + "äåéë".encode("latin_1")) """ ) fn = partial(pythons, py.name, _encoding="ascii") self.assertRaises(UnicodeDecodeError, fn) p = pythons(py.name, _encoding="ascii", _decode_errors="ignore") self.assertEqual(p, "test") p = pythons( py.name, _encoding="ascii", _decode_errors="ignore", _out=sys.stdout, _tee=True, ) self.assertEqual(p, "test") def test_signal_exception(self): from sh import SignalException_15 def throw_terminate_signal(): py = create_tmp_test( """ import time while True: time.sleep(1) """ ) to_kill = python(py.name, _bg=True) to_kill.terminate() to_kill.wait() self.assertRaises(SignalException_15, throw_terminate_signal) def test_signal_group(self): child = create_tmp_test( """ import time time.sleep(3) """ ) parent = create_tmp_test( """ import sys import sh python = sh.Command(sys.executable) p = python("{child_file}", _bg=True, _new_session=False) print(p.pid) print(p.process.pgid) p.wait() """, child_file=child.name, ) def launch(): p = python(parent.name, _bg=True, _iter=True, _new_group=True) child_pid = int(next(p).strip()) child_pgid = int(next(p).strip()) parent_pid = p.pid parent_pgid = p.process.pgid return p, child_pid, child_pgid, parent_pid, parent_pgid def assert_alive(pid): os.kill(pid, 0) def assert_dead(pid): self.assert_oserror(errno.ESRCH, os.kill, pid, 0) # first let's prove that calling regular SIGKILL on the parent does # nothing to the child, since the child was launched in the same process # group (_new_session=False) and the parent is not a controlling process p, child_pid, child_pgid, parent_pid, parent_pgid = launch() assert_alive(parent_pid) assert_alive(child_pid) p.kill() time.sleep(0.1) assert_dead(parent_pid) assert_alive(child_pid) self.assertRaises(sh.SignalException_SIGKILL, p.wait) assert_dead(child_pid) # now let's prove that killing the process group kills both the parent # and the child p, child_pid, child_pgid, parent_pid, parent_pgid = launch() assert_alive(parent_pid) assert_alive(child_pid) p.kill_group() time.sleep(0.1) assert_dead(parent_pid) assert_dead(child_pid) def test_pushd(self): """test basic pushd functionality""" child = realpath(tempfile.mkdtemp()) old_wd1 = sh.pwd().strip() old_wd2 = os.getcwd() self.assertEqual(old_wd1, old_wd2) self.assertNotEqual(old_wd1, child) with sh.pushd(child): new_wd1 = sh.pwd().strip() new_wd2 = os.getcwd() old_wd3 = sh.pwd().strip() old_wd4 = os.getcwd() self.assertEqual(old_wd3, old_wd4) self.assertEqual(old_wd1, old_wd3) self.assertEqual(new_wd1, child) self.assertEqual(new_wd2, child) def test_pushd_cd(self): """test that pushd works like pushd/popd""" child = realpath(tempfile.mkdtemp()) try: old_wd = os.getcwd() with sh.pushd(tempdir): self.assertEqual(str(tempdir), os.getcwd()) self.assertEqual(old_wd, os.getcwd()) finally: os.rmdir(child) def test_non_existant_cwd(self): from sh import ls # sanity check non_exist_dir = join(tempdir, "aowjgoahewro") self.assertFalse(exists(non_exist_dir)) self.assertRaises(sh.ForkException, ls, _cwd=non_exist_dir) # https://github.com/amoffat/sh/issues/176 def test_baked_command_can_be_printed(self): from sh import ls ll = ls.bake("-l") self.assertTrue(str(ll).endswith("/ls -l")) def test_baked_command_can_be_printed_with_whitespace_args(self): from sh import ls ls_himym = ls.bake("How I Met Your Mother") self.assertTrue(str(ls_himym).endswith("/ls 'How I Met Your Mother'")) ls_himym = ls.bake("How I 'Met' Your Mother") self.assertTrue( str(ls_himym).endswith("""/ls 'How I '"'"'Met'"'"' Your Mother'""") ) ls_himym = ls.bake('How I "Met" Your Mother') self.assertTrue(str(ls_himym).endswith("""/ls 'How I "Met" Your Mother'""")) def test_baked_command_can_be_printed_with_whitespace_in_options(self): from sh import ls cmd = ls.bake(o="one two") self.assertTrue(str(cmd).endswith("""/ls -o 'one two'"""), str(cmd)) cmd = ls.bake(opt="one two") self.assertTrue(str(cmd).endswith("""/ls '--opt=one two'"""), str(cmd)) # https://github.com/amoffat/sh/issues/185 def test_done_callback(self): import time class Callback: def __init__(self): self.called = False self.exit_code = None self.success = None def __call__(self, p, success, exit_code): self.called = True self.exit_code = exit_code self.success = success py = create_tmp_test( """ from time import time, sleep sleep(1) print(time()) """ ) callback = Callback() p = python(py.name, _done=callback, _bg=True) # do a little setup to prove that a command with a _done callback is run # in the background wait_start = time.time() p.wait() wait_elapsed = time.time() - wait_start self.assertTrue(callback.called) self.assertLess(abs(wait_elapsed - 1.0), 1.0) self.assertEqual(callback.exit_code, 0) self.assertTrue(callback.success) # https://github.com/amoffat/sh/issues/564 def test_done_callback_no_deadlock(self): import time py = create_tmp_test( """ from sh import sleep def done(cmd, success, exit_code): print(cmd, success, exit_code) sleep('1', _done=done) """ ) p = python(py.name, _bg=True, _timeout=2) # do a little setup to prove that a command with a _done callback is run # in the background wait_start = time.time() p.wait() wait_elapsed = time.time() - wait_start self.assertLess(abs(wait_elapsed - 1.0), 1.0) def test_fork_exc(self): from sh import ForkException py = create_tmp_test("") def fail(): raise RuntimeError("nooo") self.assertRaises(ForkException, python, py.name, _preexec_fn=fail) def test_new_session_new_group(self): from threading import Event py = create_tmp_test( """ import os import time pid = os.getpid() pgid = os.getpgid(pid) sid = os.getsid(pid) stuff = [pid, pgid, sid] print(",".join([str(el) for el in stuff])) time.sleep(0.5) """ ) event = Event() def handle(run_asserts, line, stdin, p): pid, pgid, sid = line.strip().split(",") pid = int(pid) pgid = int(pgid) sid = int(sid) test_pid = os.getpgid(os.getpid()) self.assertEqual(p.pid, pid) self.assertEqual(p.pgid, pgid) self.assertEqual(pgid, p.get_pgid()) self.assertEqual(p.sid, sid) self.assertEqual(sid, p.get_sid()) run_asserts(pid, pgid, sid, test_pid) event.set() def session_true_group_false(pid, pgid, sid, test_pid): self.assertEqual(pid, sid) self.assertEqual(pid, pgid) p = python( py.name, _out=partial(handle, session_true_group_false), _new_session=True ) p.wait() self.assertTrue(event.is_set()) event.clear() def session_false_group_false(pid, pgid, sid, test_pid): self.assertEqual(test_pid, pgid) self.assertNotEqual(pid, sid) p = python( py.name, _out=partial(handle, session_false_group_false), _new_session=False ) p.wait() self.assertTrue(event.is_set()) event.clear() def session_false_group_true(pid, pgid, sid, test_pid): self.assertEqual(pid, pgid) self.assertNotEqual(pid, sid) p = python( py.name, _out=partial(handle, session_false_group_true), _new_session=False, _new_group=True, ) p.wait() self.assertTrue(event.is_set()) event.clear() def test_done_cb_exc(self): from sh import ErrorReturnCode class Callback: def __init__(self): self.called = False self.success = None def __call__(self, p, success, exit_code): self.success = success self.called = True py = create_tmp_test("exit(1)") callback = Callback() try: p = python(py.name, _done=callback, _bg=True) p.wait() except ErrorReturnCode: self.assertTrue(callback.called) self.assertFalse(callback.success) else: self.fail("command should've thrown an exception") def test_callable_stdin(self): py = create_tmp_test( """ import sys sys.stdout.write(sys.stdin.read()) """ ) def create_stdin(): state = {"count": 0} def stdin(): count = state["count"] if count == 4: return None state["count"] += 1 return str(count) return stdin out = pythons(py.name, _in=create_stdin()) self.assertEqual("0123", out) def test_stdin_unbuffered_bufsize(self): from time import sleep # this tries to receive some known data and measures the time it takes # to receive it. since we're flushing by newline, we should only be # able to receive the data when a newline is fed in py = create_tmp_test( """ import sys from time import time started = time() data = sys.stdin.read(len("testing")) waited = time() - started sys.stdout.write(data + "\\n") sys.stdout.write(str(waited) + "\\n") started = time() data = sys.stdin.read(len("done")) waited = time() - started sys.stdout.write(data + "\\n") sys.stdout.write(str(waited) + "\\n") sys.stdout.flush() """ ) def create_stdin(): yield "test" sleep(1) yield "ing" sleep(1) yield "done" out = python(py.name, _in=create_stdin(), _in_bufsize=0) word1, time1, word2, time2, _ = out.split("\n") time1 = float(time1) time2 = float(time2) self.assertEqual(word1, "testing") self.assertLess(abs(1 - time1), 0.5) self.assertEqual(word2, "done") self.assertLess(abs(1 - time2), 0.5) def test_stdin_newline_bufsize(self): from time import sleep # this tries to receive some known data and measures the time it takes # to receive it. since we're flushing by newline, we should only be # able to receive the data when a newline is fed in py = create_tmp_test( """ import sys from time import time started = time() data = sys.stdin.read(len("testing\\n")) waited = time() - started sys.stdout.write(data) sys.stdout.write(str(waited) + "\\n") started = time() data = sys.stdin.read(len("done\\n")) waited = time() - started sys.stdout.write(data) sys.stdout.write(str(waited) + "\\n") sys.stdout.flush() """ ) # we'll feed in text incrementally, sleeping strategically before # sending a newline. we then measure the amount that we slept # indirectly in the child process def create_stdin(): yield "test" sleep(1) yield "ing\n" sleep(1) yield "done\n" out = python(py.name, _in=create_stdin(), _in_bufsize=1) word1, time1, word2, time2, _ = out.split("\n") time1 = float(time1) time2 = float(time2) self.assertEqual(word1, "testing") self.assertLess(abs(1 - time1), 0.5) self.assertEqual(word2, "done") self.assertLess(abs(1 - time2), 0.5) def test_custom_timeout_signal(self): import signal from sh import TimeoutException py = create_tmp_test( """ import time time.sleep(3) """ ) try: python(py.name, _timeout=1, _timeout_signal=signal.SIGHUP) except TimeoutException as e: self.assertEqual(e.exit_code, signal.SIGHUP) else: self.fail("we should have handled a TimeoutException") def test_timeout_race_condition_process_exit(self): import signal from sh import TimeoutException py = create_tmp_test( """ import time time.sleep(0.05) """ ) # Run multiple times to increase likelihood of hitting the race condition for _ in range(50): try: python(py.name, _timeout=0.1, _timeout_signal=signal.SIGTERM) except TimeoutException: # TimeoutException is OK, but ProcessLookupError isn't pass def test_append_stdout(self): py = create_tmp_test( """ import sys num = sys.stdin.read() sys.stdout.write(num) """ ) append_file = tempfile.NamedTemporaryFile(mode="a+b") python(py.name, _in="1", _out=append_file) python(py.name, _in="2", _out=append_file) append_file.seek(0) output = append_file.read() self.assertEqual(b"12", output) def test_shadowed_subcommand(self): py = create_tmp_test( """ import sys sys.stdout.write(sys.argv[1]) """ ) out = pythons.bake(py.name).bake_() self.assertEqual("bake", out) def test_no_proc_no_attr(self): py = create_tmp_test("") with python(py.name) as p: self.assertRaises(AttributeError, getattr, p, "exit_code") def test_partially_applied_callback(self): from functools import partial py = create_tmp_test( """ for i in range(10): print(i) """ ) output = [] def fn(foo, line): output.append((foo, int(line.strip()))) log_line = partial(fn, "hello") python(py.name, _out=log_line) self.assertEqual(output, [("hello", i) for i in range(10)]) output = [] def fn(foo, line, stdin, proc): output.append((foo, int(line.strip()))) log_line = partial(fn, "hello") python(py.name, _out=log_line) self.assertEqual(output, [("hello", i) for i in range(10)]) # https://github.com/amoffat/sh/issues/266 def test_grandchild_no_sighup(self): import time # child process that will write to a file if it receives a SIGHUP child = create_tmp_test( """ import signal import sys import time output_file = sys.argv[1] with open(output_file, "w") as f: def handle_sighup(signum, frame): f.write("got signal %d" % signum) sys.exit(signum) signal.signal(signal.SIGHUP, handle_sighup) time.sleep(2) f.write("made it!\\n") """ ) # the parent that will terminate before the child writes to the output # file, potentially causing a SIGHUP parent = create_tmp_test( """ import os import time import sys child_file = sys.argv[1] output_file = sys.argv[2] python_name = os.path.basename(sys.executable) os.spawnlp(os.P_NOWAIT, python_name, python_name, child_file, output_file) time.sleep(1) # give child a chance to set up """ ) output_file = tempfile.NamedTemporaryFile(delete=True) python(parent.name, child.name, output_file.name) time.sleep(3) out = output_file.readlines()[0] self.assertEqual(out, b"made it!\n") def test_unchecked_producer_failure(self): from sh import ErrorReturnCode_2 producer = create_tmp_test( """ import sys for i in range(10): print(i) sys.exit(2) """ ) consumer = create_tmp_test( """ import sys for line in sys.stdin: pass """ ) direct_pipe = python(producer.name, _piped=True) self.assertRaises(ErrorReturnCode_2, python, direct_pipe, consumer.name) def test_unchecked_pipeline_failure(self): # similar to test_unchecked_producer_failure, but this # tests a multi-stage pipeline from sh import ErrorReturnCode_2 producer = create_tmp_test( """ import sys for i in range(10): print(i) sys.exit(2) """ ) middleman = create_tmp_test( """ import sys for line in sys.stdin: print("> " + line) """ ) consumer = create_tmp_test( """ import sys for line in sys.stdin: pass """ ) producer_normal_pipe = python(producer.name, _piped=True) middleman_normal_pipe = python( middleman.name, _piped=True, _in=producer_normal_pipe ) self.assertRaises( ErrorReturnCode_2, python, middleman_normal_pipe, consumer.name ) def test_bad_sig_raise_exception(self): # test all bad signal are correctly raised py = create_tmp_test( """ import time import sys time.sleep(2) sys.exit(1) """ ) for sig in SIGNALS_THAT_SHOULD_THROW_EXCEPTION: if sig == signal.SIGPIPE: continue sig_exception_name = f"SignalException_{sig}" sig_exception = getattr(sh, sig_exception_name) try: p = python_bg(py.name) time.sleep(0.5) p.signal(sig) p.wait() except sig_exception: pass else: self.fail(f"{sig_exception_name} not raised") def test_ok_code_ignores_bad_sig_exception(self): # Test if I have [-sig] in _ok_code, the exception won't be raised py = create_tmp_test( """ import time import sys time.sleep(2) sys.exit(1) """ ) for sig in SIGNALS_THAT_SHOULD_THROW_EXCEPTION: if sig == signal.SIGPIPE: continue sig_exception_name = f"SignalException_{sig}" sig_exception = getattr(sh, sig_exception_name) python_bg_no_sig_exception = python_bg.bake(_ok_code=[-sig]) try: p = python_bg_no_sig_exception(py.name) time.sleep(0.5) p.signal(sig) p.wait() except sig_exception: self.fail( f"{sig_exception_name} should not be raised setting _ok_code." ) else: self.assertEqual(p.exit_code, -sig) class MockTests(BaseTests): def test_patch_command_cls(self): def fn(): cmd = sh.Command("afowejfow") return cmd() @unittest.mock.patch("sh.Command") def test(Command): Command().return_value = "some output" return fn() self.assertEqual(test(), "some output") self.assertRaises(sh.CommandNotFound, fn) def test_patch_command(self): def fn(): return sh.afowejfow() @unittest.mock.patch("sh.afowejfow", create=True) def test(cmd): cmd.return_value = "some output" return fn() self.assertEqual(test(), "some output") self.assertRaises(sh.CommandNotFound, fn) class MiscTests(BaseTests): def test_pickling(self): import pickle py = create_tmp_test( """ import sys sys.stdout.write("some output") sys.stderr.write("some error") exit(1) """ ) try: python(py.name) except sh.ErrorReturnCode as e: restored = pickle.loads(pickle.dumps(e)) self.assertEqual(restored.stdout, b"some output") self.assertEqual(restored.stderr, b"some error") self.assertEqual(restored.exit_code, 1) else: self.fail("Didn't get an exception") @requires_poller("poll") def test_fd_over_1024(self): py = create_tmp_test("""print("hi world")""") with ulimit(resource.RLIMIT_NOFILE, 2048): cutoff_fd = 1024 pipes = [] for i in range(cutoff_fd): master, slave = os.pipe() pipes.append((master, slave)) if slave >= cutoff_fd: break python(py.name) for master, slave in pipes: os.close(master) os.close(slave) def test_args_deprecated(self): self.assertRaises(DeprecationWarning, sh.args, _env={}) def test_percent_doesnt_fail_logging(self): """test that a command name doesn't interfere with string formatting in the internal loggers""" py = create_tmp_test( """ print("cool") """ ) python(py.name, "%") python(py.name, "%%") python(py.name, "%%%") def test_pushd_thread_safety(self): import threading import time temp1 = realpath(tempfile.mkdtemp()) temp2 = realpath(tempfile.mkdtemp()) try: results = [None, None] def fn1(): with sh.pushd(temp1): time.sleep(0.2) results[0] = realpath(os.getcwd()) def fn2(): time.sleep(0.1) with sh.pushd(temp2): results[1] = realpath(os.getcwd()) time.sleep(0.3) t1 = threading.Thread(name="t1", target=fn1) t2 = threading.Thread(name="t2", target=fn2) t1.start() t2.start() t1.join() t2.join() self.assertEqual(results, [temp1, temp2]) finally: os.rmdir(temp1) os.rmdir(temp2) def test_stdin_nohang(self): py = create_tmp_test( """ print("hi") """ ) read, write = os.pipe() stdin = os.fdopen(read, "r") python(py.name, _in=stdin) @requires_utf8 def test_unicode_path(self): from sh import Command python_name = os.path.basename(sys.executable) py = create_tmp_test( f"""#!/usr/bin/env {python_name} # -*- coding: utf8 -*- print("字") """, prefix="字", delete=False, ) try: py.close() os.chmod(py.name, int(0o755)) cmd = Command(py.name) # all of these should behave just fine str(cmd) repr(cmd) running = cmd(_return_cmd=True) str(running) repr(running) str(running.process) repr(running.process) finally: os.unlink(py.name) # https://github.com/amoffat/sh/issues/121 def test_wraps(self): from sh import ls wraps(ls)(lambda f: True) def test_signal_exception_aliases(self): """proves that signal exceptions with numbers and names are equivalent""" import signal import sh sig_name = f"SignalException_{signal.SIGQUIT}" sig = getattr(sh, sig_name) from sh import SignalException_SIGQUIT self.assertEqual(sig, SignalException_SIGQUIT) def test_change_log_message(self): py = create_tmp_test( """ print("cool") """ ) def log_msg(cmd, call_args, pid=None): return "Hi! I ran something" buf = StringIO() handler = logging.StreamHandler(buf) logger = logging.getLogger("sh") logger.setLevel(logging.INFO) try: logger.addHandler(handler) python(py.name, "meow", "bark", _log_msg=log_msg) finally: logger.removeHandler(handler) loglines = buf.getvalue().split("\n") self.assertTrue(loglines, "Log handler captured no messages?") self.assertTrue(loglines[0].startswith("Hi! I ran something")) # https://github.com/amoffat/sh/issues/273 def test_stop_iteration_doesnt_block(self): """proves that calling calling next() on a stopped iterator doesn't hang.""" py = create_tmp_test( """ print("cool") """ ) p = python(py.name, _iter=True) for i in range(100): try: next(p) except StopIteration: pass # https://github.com/amoffat/sh/issues/195 def test_threaded_with_contexts(self): import threading import time py = create_tmp_test( """ import sys a = sys.argv res = (a[1], a[3]) sys.stdout.write(repr(res)) """ ) p1 = python.bake("-u", py.name, 1) p2 = python.bake("-u", py.name, 2) results = [None, None] def f1(): with p1: time.sleep(1) results[0] = str(system_python("one")) def f2(): with p2: results[1] = str(system_python("two")) t1 = threading.Thread(target=f1) t1.start() t2 = threading.Thread(target=f2) t2.start() t1.join() t2.join() correct = [ "('1', 'one')", "('2', 'two')", ] self.assertEqual(results, correct) # https://github.com/amoffat/sh/pull/292 def test_eintr(self): import signal def handler(num, frame): pass signal.signal(signal.SIGALRM, handler) py = create_tmp_test( """ import time time.sleep(2) """ ) p = python(py.name, _bg=True) signal.alarm(1) p.wait() class StreamBuffererTests(unittest.TestCase): def test_unbuffered(self): from sh import StreamBufferer b = StreamBufferer(0) self.assertEqual(b.process(b"test"), [b"test"]) self.assertEqual(b.process(b"one"), [b"one"]) self.assertEqual(b.process(b""), [b""]) self.assertEqual(b.flush(), b"") def test_newline_buffered(self): from sh import StreamBufferer b = StreamBufferer(1) self.assertEqual(b.process(b"testing\none\ntwo"), [b"testing\n", b"one\n"]) self.assertEqual(b.process(b"\nthree\nfour"), [b"two\n", b"three\n"]) self.assertEqual(b.flush(), b"four") def test_chunk_buffered(self): from sh import StreamBufferer b = StreamBufferer(10) self.assertEqual(b.process(b"testing\none\ntwo"), [b"testing\non"]) self.assertEqual(b.process(b"\nthree\n"), [b"e\ntwo\nthre"]) self.assertEqual(b.flush(), b"e\n") @requires_posix class ExecutionContextTests(unittest.TestCase): def test_basic(self): import sh py = create_tmp_test( """ import sys sys.stdout.write(sys.argv[1]) """ ) out = StringIO() sh2 = sh.bake(_out=out) sh2.python(py.name, "TEST") self.assertEqual("TEST", out.getvalue()) def test_multiline_defaults(self): py = create_tmp_test( """ import os print(os.environ["ABC"]) """ ) sh2 = sh.bake( _env={ "ABC": "123", } ) output = sh2.python(py.name).strip() assert output == "123" def test_no_interfere1(self): import sh py = create_tmp_test( """ import sys sys.stdout.write(sys.argv[1]) """ ) out = StringIO() _sh = sh.bake(_out=out) # noqa: F841 _sh.python(py.name, "TEST") self.assertEqual("TEST", out.getvalue()) # Emptying the StringIO out.seek(0) out.truncate(0) sh.python(py.name, "KO") self.assertEqual("", out.getvalue()) def test_no_interfere2(self): import sh out = StringIO() from sh import echo _sh = sh.bake(_out=out) # noqa: F841 echo("-n", "TEST") self.assertEqual("", out.getvalue()) def test_set_in_parent_function(self): import sh py = create_tmp_test( """ import sys sys.stdout.write(sys.argv[1]) """ ) out = StringIO() _sh = sh.bake(_out=out) def nested1(): _sh.python(py.name, "TEST1") def nested2(): import sh sh.python(py.name, "TEST2") nested1() nested2() self.assertEqual("TEST1", out.getvalue()) def test_command_with_baked_call_args(self): # Test that sh.Command() knows about baked call args import sh _sh = sh.bake(_ok_code=1) self.assertEqual(sh.Command._call_args["ok_code"], 0) self.assertEqual(_sh.Command._call_args["ok_code"], 1) if __name__ == "__main__": root = logging.getLogger() root.setLevel(logging.DEBUG) root.addHandler(logging.NullHandler()) test_kwargs = {"warnings": "ignore"} # if we're running a specific test, we can let unittest framework figure out # that test and run it itself. it will also handle setting the return code # of the process if any tests error or fail if len(sys.argv) > 1: unittest.main(**test_kwargs) # otherwise, it looks like we want to run all the tests else: suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__]) test_kwargs["verbosity"] = 2 result = unittest.TextTestRunner(**test_kwargs).run(suite) if not result.wasSuccessful(): exit(1) ================================================ FILE: tox.ini ================================================ [tox] envlist = py{38,39,310,311}-locale-{c,utf8}-poller-{poll,select},lint isolated_build = True [testenv] allowlist_externals = poetry setenv = locale-c: LANG=C locale-utf8: LANG=en_US.UTF-8 poller-select: SH_TESTS_USE_SELECT=1 poller-poll: SH_TESTS_USE_SELECT=0 SH_TESTS_RUNNING=1 commands = python sh_test.py {posargs} [testenv:lint] allowlist_externals = flake8 black rstcheck mypy commands = flake8 sh.py sh_test.py black --check --diff sh.py sh_test.py rstcheck README.rst mypy sh.py