Repository: flaggo/pydu Branch: master Commit: e6e4055f81db Files: 144 Total size: 200.9 KB Directory structure: gitextract_0sr93e7l/ ├── .appveyor.yml ├── .coveragerc ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── README.md ├── docs/ │ ├── .nojekyll │ ├── README.md │ ├── _coverpage.md │ ├── _navbar.md │ ├── _sidebar.md │ ├── archive.md │ ├── cmd.md │ ├── compat.md │ ├── console.md │ ├── convert.md │ ├── dict.md │ ├── dt.md │ ├── environ.md │ ├── exception.md │ ├── functional.md │ ├── index.html │ ├── inspect.md │ ├── iter.md │ ├── list.md │ ├── misc.md │ ├── network.md │ ├── path.md │ ├── platform.md │ ├── process.md │ ├── request.md │ ├── set.md │ ├── slot.md │ ├── string.md │ ├── system.md │ ├── unit.md │ └── zh-cn/ │ ├── README.md │ ├── _sidebar.md │ ├── archive.md │ ├── cmd.md │ ├── compat.md │ ├── console.md │ ├── convert.md │ ├── dict.md │ ├── dt.md │ ├── environ.md │ ├── exception.md │ ├── functional.md │ ├── inspect.md │ ├── iter.md │ ├── list.md │ ├── misc.md │ ├── network.md │ ├── path.md │ ├── platform.md │ ├── process.md │ ├── request.md │ ├── set.md │ ├── slot.md │ ├── string.md │ ├── system.md │ └── unit.md ├── pydu/ │ ├── __init__.py │ ├── archive.py │ ├── cmd.py │ ├── compat.py │ ├── console.py │ ├── convert.py │ ├── dict.py │ ├── dt.py │ ├── environ.py │ ├── exception.py │ ├── functional.py │ ├── inspect.py │ ├── iter.py │ ├── list.py │ ├── misc.py │ ├── network.py │ ├── path.py │ ├── platform.py │ ├── process.py │ ├── request.py │ ├── set.py │ ├── slot.py │ ├── string.py │ ├── system.py │ └── unit.py ├── requirements-dev.txt ├── setup.cfg ├── setup.py ├── stubs/ │ └── pydu/ │ ├── __init__.pyi │ ├── archive.pyi │ ├── cmd.pyi │ ├── console.pyi │ ├── convert.pyi │ ├── dict.pyi │ ├── dt.pyi │ ├── environ.pyi │ ├── exception.pyi │ ├── functional.pyi │ ├── iter.pyi │ ├── list.pyi │ ├── misc.pyi │ ├── network.pyi │ ├── path.pyi │ ├── process.pyi │ ├── request.pyi │ ├── set.pyi │ ├── string.pyi │ ├── system.pyi │ └── unit.pyi ├── tests/ │ ├── __init__.py │ ├── files/ │ │ ├── bad/ │ │ │ └── unrecognized.txt │ │ ├── foobar.tar.bz2 │ │ ├── foobar_tar_gz │ │ └── 压缩.tgz │ ├── test_archive.py │ ├── test_cmd.py │ ├── test_compat.py │ ├── test_console.py │ ├── test_convert.py │ ├── test_dict.py │ ├── test_dt.py │ ├── test_environ.py │ ├── test_exception.py │ ├── test_functional.py │ ├── test_inspect.py │ ├── test_iter.py │ ├── test_list.py │ ├── test_misc.py │ ├── test_network.py │ ├── test_path.py │ ├── test_platform.py │ ├── test_request.py │ ├── test_set.py │ ├── test_slot.py │ ├── test_string.py │ ├── test_system.py │ ├── test_unit.py │ └── testing.py └── tox.ini ================================================ FILE CONTENTS ================================================ ================================================ FILE: .appveyor.yml ================================================ build: off environment: matrix: - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "64" TOXENV: "py27" - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "64" TOXENV: "py35" - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" TOXENV: "py36" - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7.x" PYTHON_ARCH: "64" TOXENV: "py37" - PYTHON: "C:\\Python38-x64" PYTHON_VERSION: "3.8.x" PYTHON_ARCH: "64" TOXENV: "py38" install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "%CMD_IN_ENV% pip install tox codecov" test_script: - "%CMD_IN_ENV% tox" on_success: - "%CMD_IN_ENV% codecov" ================================================ FILE: .coveragerc ================================================ [run] branch = True source = pydu [paths] source = pydu .tox/*/lib/python*/site-packages/pydu ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # IDE .idea # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # dotenv .env # virtualenv .venv venv/ ENV/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ # pytest .pytest_cache/ # Mac .DS_Store ================================================ FILE: .travis.yml ================================================ language: python python: - "2.7" - "3.5" - "3.6" - "3.7" - "3.8" sudo: false cache: pip install: - pip install tox codecov script: - tox -e $(echo py$TRAVIS_PYTHON_VERSION | tr -d .) after_success: - codecov notifications: email: recipients: - wangbinxin001@126.com on_success: always on_failure: always ================================================ FILE: CHANGELOG.md ================================================ v0.7.2 (2019-02-08) ------------------- **Bug fixes** * Fix collections ABCs deprecation warning v0.7.0 (2018-05-14) ------------------- **Enhancements** * Upgrade to **brand new document** powerd by docsify * Add slot.SlotBase which is the base class for class using `__slots__` * Add compat.izip v0.6.2 (2018-04-30) ------------------- **Enhancements** * Add ``exception.default_if_except`` which excepts given exceptions and return default value as decorator. v0.6.1 (2018-04-23) ------------------- **Enhancements** * Add ``dt.timer`` which can time how long does calling take as a context manager or decorator. v0.6.0 (2018-04-16) ------------------- **Enhancements** * Add ``path.filename`` which return the filename without extension. * Add ``path.fileext`` which return the file extension. * Update stub for ``requests.check_connect``. v0.5.2 (2018-04-04) ------------------- **Enhancements** * Add ``system.preferredencoding`` which gets best encoding for the system. * Add ``request.update_query_params`` which update query params of given url and return new url. * Update stub for ``requests.check_connect``. v0.5.1 (2018-03-19) ------------------- **Enhancements** * Improve ``system.remove`` when path is read-only. * Add ``path.normjoin`` which join one or more path components intelligently and normalize it. * Improve ``environ.environ`` with supporting variable_name=None which means removing the variable from environment temporarily. v0.5.0 (2018-03-08) ------------------- **Enhancements** * Add ``network.private_ipv4s`` which stores private IPV4 addresses. * Add ``functional.compose`` which composes all functions into one. * Add ``TYPE HINT`` for ALL MODULES by supplying STUB FILES! **Bug fixes** * Fix reduce error on Python 3. v0.4.2 (2018-02-05) ------------------- **Enhancements** * Add ``socket.inet_pton`` and ``socket.inetntop`` for Windows if we ``import pydu.network``. * Add ``network.ip2int`` and ``network.int2ip`` which convert ip to integer or integer to ip. * Add ``process.get_processes_by_path`` for getting processes which are running on given path or sub path of given path. * Add ``first``, ``last``, ``all``, ``any`` and ``join`` to ``pydu.iter``, which support many operations on iterable object. **Bug fixes** * Fix several convert functions return values with unnecessary value 'L' when given big number on Python 2. v0.4.1 (2018-01-20) ------------------- **Enhancements** * Add ``bin2oct``, ``bin2dec``, ``bin2hex``, ``oct2bin``, ``oct2dec``, ``oct2hex``, ``dec2bin``, ``dec2oct``, ``dec2hex``, ``hex2bin``, ``hex2oct``, ``hex2dec`` to ``convert``, which support many base conversions * Add ``path.is_super_path`` which judges whether the given ``path1`` is the super path of ``path2`` * Add ``environ.environ`` which is a context manager for updating one or more environment variables * Add ``environ.path`` which is a context manager for updating the PATH environment variable * Add ``list.tolist`` which converts obj to list * Add ``list.flatten`` which generates each element of the given ``seq`` * Add ``compat.strbytes_types`` which includes all types about string v0.4.0 (2018-01-09) ------------------- **Importance** * Remove support for Python 3.4 **Enhancements** * Add ``dict.OrderedDefaultDict`` which remembers insertion order and has default value with default factory * Add ``convert.boolean`` which converts obj to a boolean value * ``console.console_size`` will use ``shutil.get_terminal_size`` if possible * ``exception.ignore`` is same to ``context.lib.suppress`` on Python 3 **Bug fixes** * Fix #15 (If the ``dict.attrify``'s obj is tuple, this will raise a error) v0.3.1 (2017-12-29) ------------------- **Enhancements** * Add ``FileTracker`` which could track opening files. **Bug fixes** * Fix ``pip install`` error on Windows with Python 3. * Fix ``network.is_ipv6`` test error on Windows with Python 3. * Fix description error on ``network``, ``request`` doc. v0.3.0 (2017-12-26) ------------------- **Enhancements** * Rename ``file`` to ``system``. * Add ``system.which`` which supports find executable file. * Add ``system.chmod`` which supports chmod recursively. * Add ``unit.Bytes`` which used to deal with bytes. * Add ``preferredencoding`` to ``string``. * Add ``cmd.chcp`` for Windows which is same like ``chcp`` on Windows cmd. * Add ``cmd.run_with_en_env`` which ensure the output of cmd is in English. * Add ``cmd.terminate`` which supports terminate process by given ``pid``. * ``cmd.run`` uses timeout feature on Python 3 but not implement by self. **Bug fixes** * Fix test cases to generate right coverage. v0.2.0 (2017-12-17) ------------------- **Enhancements** * Add ``exception.ignore``. * ``network.is_ipv6`` is available on Windows. * Set logging handler to avoid "No handler found" warnings. * Add ``Makefile`` which make development easier. * Update ``readme`` which is more readable. **Bug fixes** * Fix installation error on Windows. v0.1.0 (2017-12-14) ------------------- Supply many powerful data structures and utils about archive, cmd, compat, console, dict, file, inspect, list, misc, network, path, platform, request, set and string. ================================================ FILE: LICENSE.txt ================================================ MIT License Copyright (c) 2017 Prodesire Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: MANIFEST.in ================================================ include requirements-dev.txt README.md CHANGELOG.md LICENSE.txt ================================================ FILE: Makefile ================================================ # Env export PYTHONDONTWRITEBYTECODE=1 TEST_PATH=./tests DEFAULT_PYTHON2=`python -c "import sys;print(sys.version_info.major)" | grep 2` PY2=$(if $(DEFAULT_PYTHON2),python,python2) PY3=$(if $(DEFAULT_PYTHON2),python3,python) # Func .PHONY: docs help: @echo "\033[32minit\033[0m" @echo " Init environment for pydu." @echo "\033[32mtest\033[0m" @echo " Run pytest with Python 2 and 3." @echo "\033[32mtest-py2\033[0m" @echo " Run pytest with Python 2." @echo "\033[32mtest-py3\033[0m" @echo " Run pytest with Python 3." @echo "\033[32mcoverage\033[0m" @echo " Run pytest and report coverage." @echo "\033[32mpublish\033[0m" @echo " Publish pydu to PyPI." @echo "\033[32mdocs\033[0m" @echo " Make docs for pydu." @echo "\033[32mclean\033[0m" @echo " Remove python and build artifacts." @echo "\033[32mclean-pyc\033[0m" @echo " Remove python artifacts." @echo "\033[32mclean-build\033[0m" @echo " Remove build artifacts." init: pip install -r requirements-dev.txt npm i docsify-cli -g test: test-py2 test-py3 test-py2: clean-pyc $(PY2) -m pytest --color=yes $(TEST_PATH) test-py3: clean-pyc $(PY3) -m pytest --color=yes $(TEST_PATH) coverage: coverage run --source=pydu -m pytest tests coverage report publish: pip install 'twine>=1.5.0' python setup.py sdist twine upload dist/* rm -rf build dist *.egg-info .eggs docs: docsify serve docs clean: clean-pyc clean-build clean-pyc: find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name '*~' -exec rm -f {} + find . -name '__pycache__' -exec rm -rf {} + clean-build: rm -rf build dist *.egg-info .eggs ================================================ FILE: README.md ================================================ # pydu [![pydu](https://img.shields.io/pypi/v/pydu.svg)](https://pypi.python.org/pypi/pydu) [![pydu](https://img.shields.io/pypi/l/pydu.svg)](https://pypi.python.org/pypi/pydu) [![pydu](https://img.shields.io/pypi/pyversions/pydu.svg)](https://pypi.python.org/pypi/pydu) [![pydu](https://img.shields.io/travis/flaggo/pydu/master.svg?label=Linux)](https://travis-ci.org/flaggo/pydu) [![pydu](https://img.shields.io/appveyor/ci/flaggo/pydu/master.svg?label=Windows)](https://ci.appveyor.com/project/flaggo/pydu) [![pydu](https://codecov.io/github/flaggo/pydu/coverage.svg?branch=master)](https://codecov.io/github/flaggo/pydu) [![pydu](https://img.shields.io/github/contributors/flaggo/pydu.svg)](https://github.com/flaggo/pydu/graphs/contributors) **pydu** is a library of useful **d**ata structures and **u**tils for Python 2 and 3, which collected from open source projects and created by contributors. ## Installation To install pydu, simply: ```bash $ pip install pydu ``` ## Document Fantastic documentation is available at: [English](https://flaggo.github.io/pydu/) | [中文版](https://flaggo.github.io/pydu/#/zh-cn/). ================================================ FILE: docs/.nojekyll ================================================ ================================================ FILE: docs/README.md ================================================ ## pydu > **pydu(Python Data structures and Utils)** is a library of useful data structures and utils for Python 2 and 3, which collected from open source projects and created by contributors. ## Installation To install **pydu**, simply: ```bash $ pip install pydu ``` ================================================ FILE: docs/_coverpage.md ================================================ # pydu 0.7.0 > Python Data structures and Utils. * Rich basic data structures * A variety of utils for handling different situations [GitHub](https://github.com/flaggo/pydu/) [Modules](#pydu) ================================================ FILE: docs/_navbar.md ================================================ * [En](/) * [中文](/zh-cn/) ================================================ FILE: docs/_sidebar.md ================================================ * Modules * [Archive](archive.md) * [Commad](cmd.md) * [Compat](compat.md) * [Console](console.md) * [Convert](convert.md) * [Dict](dict.md) * [Date and Time](dt.md) * [Environment](environ.md) * [Exception](exception.md) * [Functional](functional.md) * [Inspect](inspect.md) * [Iter](iter.md) * [List](list.md) * [Miscellanea](misc.md) * [Network](network.md) * [Path](path.md) * [Platform](platform.md) * [Process](process.md) * [Request](request.md) * [Set](set.md) * [Slot](slot.md) * [String](string.md) * [System](system.md) * [Unit](unit.md) * [Changelog](changelog.md) ================================================ FILE: docs/archive.md ================================================ # archive Utils for archiving files. ## archive.extract ```python extract(path, to_path='', ext='') ``` Unpack the tar or zip file at the specified path or file to the directory specified by ``to_path``. It supports many extensions, like ``.tar``, ``.tar.bz2``, ``.tar.gz``, ``.tgz``, ``.tz2``, ``.zip``. If the file name of given ``path`` doesn't contain file extension, the ``ext`` parameter can be specified one of supported extensions to indicate file type. ```python >>> from pydu.archive import extract >>> extract('foobar.tgz', '/tmp') >>> extract('foobar', '/tmp', ext='.tgz') >>> extract('foobar', '/tmp') Traceback (most recent call last): ... AttributeError: pydu.archive.UnrecognizedArchiveFormat: Path not a recognized archive format: foobar ``` ================================================ FILE: docs/cmd.md ================================================ # cmd Utils for running command and getting command line. ## cmd.TimeoutExpired ```python TimeoutExpired(cmd, timeout, output=None, stderr=None) ``` This exception is raised when the timeout expires while waiting for a child process. Attributes: cmd, output, stdout, stderr, timeout ## cmd.run ```python run(cmd, shell=False, env=None, timeout=None, timeinterval=1) ``` Run cmd based on `subprocess.Popen` and return the tuple of ``(returncode, stdout)``. Note, `stderr` is redirected to `stdout`. `shell` is same to parameter of `Popen`. If the process does not terminate after `timeout` seconds, a `TimeoutExpired` exception will be raised. `timeinterval` is workable when timeout is given on Python 2. It means process status checking interval. The child process is not killed if the timeout expires, so in order to cleanup properly a well-behaved application should kill the child process and finish communication. ```python >>> from pydu.cmd import run >>> run('echo hello') (0, b'hello\r\n') # Python 3 ``` ## cmd.run_with_en_env ```python run_with_en_env(cmd, shell=False, env=None, timeout=None, timeinterval=1) ``` Run cmd with English character sets environment, so that the output will be in English. Parameters are same with `run`. ## cmd.terminate ```python terminate(pid) ``` Terminate process by given `pid`. On Windows, using `kernel32.TerminateProcess` to kill. On other platforms, using `os.kill` with `signal.SIGTERM` to kill. ## cmd.cmdline_argv ```python cmdline_argv() ``` Get command line argv of self python process. On Windows when using Python 2, `cmdline_argv` is implemented by using `shell32.GetCommandLineArgvW` to get `sys.argv` as a list of Unicode strings. On other platforms or using Python 3, `cmdline_argv` is same to `sys.argv`. ```python >>> from pydu.cmd import cmdline_argv >>> cmdline_argv() ['/Applications/PyCharm.app/Contents/helpers/pydev/pydevconsole.py', '61253', '61254'] ``` ================================================ FILE: docs/compat.md ================================================ # compat compatible data structures, libs, functions for Python 2 and 3. ## compat.PY2 Specify current Python interpreter is Python 2 or 3. ## compat.urlib ```python urlib(base, url, allow_fragments=True) ``` Same to ``urllib`` on PY2 or ``urllib.request`` on PY3. ## compat.urlparse ```python urlparse(base, url, allow_fragments=True) ``` Same to ``urlparse`` on PY2 or ``urllib.parse`` on PY3. ## compat.urljoin ```python urljoin(base, url, allow_fragments=True) ``` Same to ``urlparse.urljoin`` on PY2 or ``urllib.parse.urljoin`` on PY3. ## compat.iterkeys ```python iterkeys(d) ``` Return an iter object of dictionary keys. ## compat.itervalues ```python itervalues(d) ``` Return an iter object of dictionary values. ## compat.iteritems ```python iteritems(d) ``` Return an iter object of dictionary items. ## compat.text_type The text type is ``unicode`` on PY2 or ``str`` on PY3. ## compat.string_types The string types are ``(str, unicode)`` on PY2 or ``(str,)`` on PY3. ## compat.strbytes_types The strbytes(string bytes) types are ``(str, unicode, bytes)`` on PY2 or ``(str, bytes)`` on PY3. ## compat.numeric_types The numeric types are ``(int, long)`` on PY2 or ``(int,)`` on PY3. ## compat.imap ```python imap(func, *iterables) ``` Same to ``itertools.imap`` on PY2 or ``map`` on PY3. ## compat.izip ```python izip(iter1 [,iter2 [...]) ``` Same to ``itertools.izip`` on PY2 or ``zip`` on PY3. ## compat.reduce ```python reduce(function, sequence, initial=None) ``` Same to built-in ``reduce`` on PY2 or ``functools.reduce`` on PY3. ## compat.cmp ```python cmp(x, y) ``` Same to ``cmp`` on PY2, but implement on PY3. ## compat.has_next_attr ```python has_next_attr(x) ``` An implementation independent way of checking for next attribute. ## compat.is_iterable ```python is_iterable(x) ``` An implementation independent way of checking for iterables. ```python >>> from pydu.compat import is_iterable >>> is_iterable([]) True >>> is_iterable(1) False ``` ================================================ FILE: docs/console.md ================================================ # Console Utils for handling console. ## console.console_size ```python console_size(fallback=(80, 25)) ``` For Windows, return (width, height) of available window area, fallback if no console is allocated. For POSIX system, return (width, height) of console terminal, fallback on IOError, i.e. when no console is allocated. For other system, return fallback. Fallback defaults to (80, 25) which is the default size used by many terminal emulators. ```python >>> from pydu.console import console_size >>> console_size() (80, 25) ``` ================================================ FILE: docs/convert.md ================================================ # Convert Utils for converting one type of data to another. ## convert.boolean ```python boolean(obj) ``` Convert obj to a boolean value. If obj is string, obj will converted by case-insensitive way: * convert `yes`, `y`, `on`, `true`, `t`, `1` to True * convert `no`, `n`, `off`, `false`, `f`, `0` to False * raising TypeError if other values passed If obj is non-string, obj will converted by `bool(obj)`. ```python >>> from pydu.string import boolean >>> boolean('yes') True >>> boolean('no') False ``` ## convert.bin2oct ```python bin2oct(x) ``` Convert binary string to octal string. For instance: '1001' -> '11' ```python >>> from pydu.convert import bin2oct >>> bin2oct('1001') '11' ``` ## convert.bin2dec ```python bin2dec(x) ``` Convert binary string to decimal number. For instance: '11' -> 3 ```python >>> from pydu.convert import bin2dec >>> bin2dec('11') 3 ``` ## convert.bin2hex ```python bin2hex(x) ``` Convert binary string to hexadecimal string. For instance: '11010' -> '1a' ```python >>> from pydu.convert import bin2hex >>> bin2hex('11010') '1a' ``` ## convert.oct2bin ```python oct2bin(x) ``` Convert octal string to binary string. For instance: '11' -> '1001' ```python >>> from pydu.convert import oct2bin >>> oct2bin('11') '1001' ``` ## convert.oct2dec ```python oct2dec(x) ``` Convert octal string to decimal number. For instance: '11' -> 9 ```python >>> from pydu.convert import oct2dec >>> oct2dec('11') 9 ``` ## convert.oct2hex ```python oct2hex(x) ``` Convert octal string to hexadecimal string. For instance: '32' -> '1a' ```python >>> from pydu.convert import oct2hex >>> oct2hex('32') '1a' ``` ## convert.dec2bin ```python dec2bin(x) ``` Convert decimal number to binary string. For instance: 3 -> '11' ```python >>> from pydu.convert import dec2bin >>> dec2bin(3) '11' ``` ## convert.dec2oct ```python dec2oct(x) ``` Convert decimal number to octal string. For instance: 9 -> '11' ```python >>> from pydu.convert import dec2oct >>> dec2oct(9) '11' ``` ## convert.dec2hex ```python dec2hex(x) ``` Convert decimal number to hexadecimal string. For instance: 26 -> '1a' ```python >>> from pydu.convert import dec2hex >>> dec2hex(26) '1a' ``` ## convert.hex2bin ```python hex2bin(x) ``` Convert hexadecimal string to binary string. For instance: '1a' -> '11010' ```python >>> from pydu.convert import hex2bin >>> hex2bin('1a') '11010' ``` ## convert.hex2oct ```python hex2oct(x) ``` Convert hexadecimal string to octal string. For instance: '1a' -> '32' ```python >>> from pydu.convert import hex2oct >>> hex2oct('1a') '32' ``` ## convert.hex2dec ```python hex2dec(x) ``` Convert hexadecimal string to decimal number. For instance: '1a' -> 26 ```python >>> from pydu.convert import hex2dec >>> hex2dec('1a') 26 ``` ================================================ FILE: docs/dict.md ================================================ # Dict Additional powerful dictionaries and relative functions. ## dict.AttrDict ```python AttrDict(seq=None, **kwargs) ``` A AttrDict object is like a dictionary except `obj.foo` can be used in addition to `obj['foo']`. ```python >>> from pydu.dict import AttrDict >>> o = AttrDict(a=1) o.a 1 >>> o['a'] 1 >>> o.a = 2 >>> o['a'] 2 >>> del o.a >>> o.a Traceback (most recent call last): ... AttributeError: 'a' ``` ## dict.CaseInsensitiveDict ```python CaseInsensitiveDict(data=None, **kwargs) ``` A case-insensitive `dict`-like object. Implements all methods and operations of `collections.MutableMapping` as well as dict's `copy`. Also provides `lower_items`. All keys are expected to be strings. The structure remembers the case of the last key to be set, and `iter(instance)`, `keys()`, `items()`, `iterkeys()`, and `iteritems()` will contain case-sensitive keys. ```python >>> from pydu.dict import CaseInsensitiveDict >>> cid = CaseInsensitiveDict() >>> cid['Accept'] = 'application/json' >>> cid['aCCEPT'] == 'application/json' True >>> list(cid) == ['Accept'] True ``` ## dict.LookupDict ```python LookupDict(name=None) ``` Dictionary lookup object. ```python >>> from pydu.dict import LookupDict >>> d = LookupDict() >>> d['key'] None >>> d['key'] = 1 >>> d['key'] 1 ``` ## dict.OrderedDefaultDict ```python OrderedDefaultDict(default_factory=None, *args, **kwds) ``` Dictionary that remembers insertion order and has default value with default factory. The default factory is called without arguments to produce a new value when a key is not present, in `__getitem__` only. An `OrderedDefaultDict` compares equal to a `collections.defaultdict` with the same items. All remaining arguments are treated the same as if they were passed to the `defaultdict` constructor, including keyword arguments. ```python >>> from pydu.dict import OrderedDefaultDict >>> d = OrderedDefaultDict(int) >>> d['b'] 0 >>> d['a'] 0 >>> d.keys() odict_keys(['b', 'a']) ``` ## dict.attrify ```python attrify(obj) ``` Attrify obj into `AttriDict` or `list of AttriDict` if the obj is list. If obj or the item of obj is not list or dict, will return itself. ```python >>> from pydu.dict import attrify >>> attrd = attrify({ 'a': [1, 2, {'b': 'b'}], 'c': 'c', }) >>> attrd ], 'c': 'c'}> >>> attrd.a 1 >>> attrd.a[2].b b >>> attrd.c c ``` ================================================ FILE: docs/dt.md ================================================ # Date and Time Utils for handling date and time. ## dt.timer ```python timer(path) ``` A timer can time how long does calling take as a context manager or decorator. If assign `print_func` with `sys.stdout.write`, `logger.info` and so on, timer will print the spent time. ```python timeit = timer(print_func=sys.stdout.write) with timeit: foo() @timeit def foo(): pass ``` `timer.elapsed` contains the total amount of elapsed time of running `foo`. ```python >>> timeit = timer(print_func=sys.stdout.write) >>> with timeit: ... os.getcwd() Spent time: 1.7881393432617188e-05s ``` ================================================ FILE: docs/environ.md ================================================ # Environ Utils for handling environment. ## environ.environ ```python environ(**kwargs) ``` Context manager for updating one or more environment variables. Preserves the previous environment variable (if available) and recovers when exiting the context manager. If given variable_name=None, it means removing the variable from environment temporarily. ```python >>> from pydu.environ import environ >>> with environ(a='a'): ... print(os.environ['a']) ... a ``` ## environ.path ```python path(append=None, prepend=None, replace=None) ``` Context manager for updating the PATH environment variable which appends, prepends or replaces the PATH with given string or a list of strings. ```python >>> import os >>> from pydu.environ import path >>> with path(append='/foo'): ... print(os.environ['PATH']) ... /usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/foo ``` ================================================ FILE: docs/exception.md ================================================ # Exception Utils for handling exceptions. ## exception.ignore ```python ignore(*exceptions) ``` A context manager which can ignore given exceptions. ```python >>> from pydu.exception import ignore >>> with ignore(ValueError, AttributeError): ... int('abc') ... int.no_exists_func() ... >>> ``` ## exception.default_if_except ```python default_if_except(exception_clses, default=None) ``` A exception decorator which excepts given exceptions and return default value. ```python >>> from pydu.exception import default_if_except >>> @default_if_except(ValueError, default=0) ... def foo(value): ... return int(value) >>> foo('abc') 0 ``` ================================================ FILE: docs/functional.md ================================================ # functional Utils for functional programming. ## functional.compose ```python compose(*funcs) ``` Compose all functions. The previous function must accept one argument, which is the output of the next function. The last function can accept any args and kwargs. `compose(f1, f2, f3)(*x)` is same to `f1(f2(f3(*x)))`. ```python >>> from pydu.functional import compose >>> def f1(a): ... return a+1 ... >>> def f2(a, b=2): ... return a+b ... >>> compose(f1, f2)(1, b=3) 5 ``` ================================================ FILE: docs/index.html ================================================ pydu - Python Data structures and Utils
Loading ...
================================================ FILE: docs/inspect.md ================================================ # inspect Utils for inspecting functions. ## inspect.getargspec ```python getargspec(func) ``` Get the names and default values of a function's arguments. A tuple of four things is returned: (args, varargs, varkw, defaults). `args` is a list of the argument names (it may contain nested lists). `varargs` and `varkw` are the names of the * and ** arguments or None. `defaults` is an n-tuple of the default values of the last n arguments. ```python >>> from pydu.inspect import getargspec >>> def f(name, address='home', age=25, *args, **kwargs): ... pass ... >>> getargspect(f) ArgSpec(args=['name', 'address', 'age'], varargs='args', keywords='kwargs', defaults=('home', 25)) ``` ## inspect.get_func_args ```python get_func_args(func) ``` Return a list of the argument names. Arguments such as `*args` and `**kwargs` are not included. ```python >>> from pydu.inspect import get_func_args >>> def f(name, address='home', age=25, *args, **kwargs): ... pass ... >>> get_func_args(f) ['name', 'address', 'age'] ``` ## inspect.get_func_full_args ```python get_func_full_args(func) ``` Return a list of (argument name, default value) tuples. If the argument does not have a default value, omit it in the tuple. Arguments such as `*args` and `**kwargs` are also included. ```python >>> from pydu.inspect import get_func_full_args >>> def f(name, address='home', age=25, *args, **kwargs): ... pass ... >>> get_func_full_args(f) [('name',), ('address', 'home'), ('age', 25), ('*args',), ('**kwargs',)] ``` ## inspect.func_accepts_kwargs ```python func_accepts_kwargs(func) ``` Check whether or not the func accepts kwargs. ```python >>> from pydu.inspect import func_accepts_kwargs >>> def f(**kwargs): ... pass ... >>> func_accepts_kwargs(f) True ``` ## inspect.func_accepts_var_args ```python func_accepts_var_args(func) ``` Check whether or not the func accepts var args. ```python >>> from pydu.inspect import func_accepts_var_args >>> def f(*vargs): ... pass ... >>> func_accepts_var_args(f) True ``` ## inspect.func_supports_parameter ```python func_supports_parameter(func) ``` Check whether or the func supports the given parameter. ```python >>> from pydu.inspect import func_supports_parameter >>> def f(name): ... pass ... >>> func_supports_parameter(f, 'name') True >>> func_supports_parameter(f, 'unkown') Fasle ``` ## inspect.func_has_no_args ```python func_has_no_args(func) ``` Check whether or not the func has any args. ```python >>> from pydu.inspect import func_has_no_args >>> def f(): ... pass ... >>> func_has_no_args(f) True ``` ================================================ FILE: docs/iter.md ================================================ # iter Utils for handling iterations. ## iter.first ```python first(iterable) ``` Get the first item in the iterable. ```python >>> from pydu.iter import first >>> first([1, 2]) 1 ``` ## iter.last ```python last(iterable) ``` Get the last item in the iterable. Warning, this can be slow due to iter step by step to last one. ```python >>> from pydu.iter import last >>> last([1, 2]) 2 ``` ## iter.all ```python all(iterable, predicate) ``` Returns True if all elements in the given iterable are True for the given predicate function. ```python >>> from pydu.iter import all >>> all([0, 1, 2], lambda x: x+1) True ``` ## iter.any ```python any(iterable) ``` Returns True if any element in the given iterable is True for the given predicate function. ```python >>> from pydu.iter import any >>> any([-1, -1, 0], lambda x: x+1) True ``` ## iter.join ```python join(iterable, separator='') ``` Join each item of iterable to string. ```python >>> from pydu.iter import join >>> join([1, '2', 3], separator=',') '1,2,3' ``` ================================================ FILE: docs/list.md ================================================ # list Utils for handling list. ## list.uniq ```python uniq(seq, key=None) ``` Removes duplicate elements from a list while preserving the order of the rest. The value of the optional `key` parameter should be a function that takes a single argument and returns a key to test the uniqueness. ```python >>> from pydu.list import uniq >>> uniq([1, 4, 0, 2, 0, 3]) [1, 4, 0, 2, 3] ``` ## list.tolist ```python tolist(obj) ``` Convert given `obj` to list. If `obj` is not a list, return `[obj]`, else return `obj` itself. ```python >>> from pydu.list import tolist >>> tolist('foo') ['foo'] ``` ## list.flatten ```python flatten(seq) ``` Generate each element of the given `seq`. If the element is iterable and is not string, it yields each sub-element of the element recursively. ```python >>> from pydu.list import flatten >>> flatten([1, [2, [3, 4]]]) [1, 2, 3, 4] ``` ================================================ FILE: docs/misc.md ================================================ # misc Miscellaneous utils like `timeout`, `trace` and so on. ## misc.timeout ```python timeout(seconds) ``` This func decorates any func which may be hang for a while. The param `seconds` can be either integer or float. In `test.py`, you may write like below: ```python import time from pydu.misc import unix_timeout @timeout(1) def f(): time.sleep(1.01) f() ``` And run `test.py`, will see `TimeoutError`. ## misc.trace ```python trace(obj) ``` Tracing every statement and line number for running program, like `bash -x`. In `test.py`, you may write like below: ```python from pydu.misc import trace @trace def f(): print(1) a = 1 + 5 b = [a] print(2) f() ``` And run `test.py`, will see below output from console: ```console test.py(4): print(1) 1 test.py(5): a = 1 + 5 test.py(6): b = [a] test.py(7): print(2) 2 ``` ## misc.memoize ```python memoize(obj) ``` A simple memoize decorator for functions supporting (hashable) positional arguments. It also provides a `cache_clear()` function for clearing the cache. ```python >>> @memoize ... def foo() ... return 1 ... >>> foo() 1 >>> foo.cache_clear() >>> ``` ## misc.memoize_when_activated ```python memoize_when_activated(obj) ``` A memoize decorator which is disabled by default. It can be activated and deactivated on request. For efficiency reasons it can be used only against class methods accepting no arguments. ```python >>> class Foo: ... @memoize ... def foo() ... print(1) ... >>> f = Foo() >>> # deactivated (default) >>> foo() 1 >>> foo() 1 >>> >>> # activated >>> foo.cache_activate() >>> foo() 1 >>> foo() >>> foo() >>> ``` ## misc.super_len ```python super_len(obj) ``` Get length of object which has attribute named `__len__`, `len`, `fileno`, `tell`, such as `list`, `tuple`, `dict`, `file` and so on. ```python >>> from pydu.misc import super_len >>> super_len([1, 2]) 2 >>> super_len(open('test', 'w')) 0 ``` ================================================ FILE: docs/network.md ================================================ # network Utils for handling network. ## network.dotted_netmask ```python dotted_netmask(mask) ``` Converts mask from /`xx` format to `xxx.xxx.xxx.xxx`. `mask` can be either `int` or `str`. ```python >>> from pydu.network import dotted_netmask >>> dotted_netmask('24') '255.255.255.0' >>> dotted_netmask(24) '255.255.255.0' ``` ## network.private_ipv4s ```python private_ipv4s ``` A list of private ipv4 addresses. Each item is a tuple of (ipv4 address, mask). ## network.is_ipv4 ```python is_ipv4(ip) ``` Judge whether the given `ip` is IPV4 address. ```python >>> from pydu.network import is_ipv4 >>> is_ipv4('8.8.8.8') True >>> is_ipv4('localhost.localdomain') False ``` ## network.is_ipv6 ```python is_ipv6(ip) ``` Judge whether the given `ip` is IPV6 address. ```python >>> from pydu.network import is_ipv6 >>> is_ipv6('fe80::9e5b:b149:e187:1a18') True >>> is_ipv6('localhost.localdomain') False ``` ## network.get_free_port ```python get_free_port() ``` Get free port which could be bound. ```python >>> from pydu.network import get_free_port >>> get_free_port() 57118 ``` ## network.ip2int ```python ip2int(ip_str) ``` Convert ip to integer. Support IPV4 and IPV6. Raise `ValueError` if convert failed. ```python >>> from pydu.network import ip2int >>> ip2int('10.1.1.1') 167837953 ``` ## network.int2ip ```python int2ip(ip_int) ``` Convert integer to ip. Support IPV4 and IPV6. Raise `ValueError` if convert failed. ```python >>> from pydu.network import int2ip >>> int2ip(167837953) '10.1.1.1' ``` ================================================ FILE: docs/path.md ================================================ # path Utils for handling path. ## path.cd ```python cd(path) ``` Context manager for cd the given path. ```python >>> from pydu.path import cd >>> with cd('test'): ... pass ``` ## path.is_super_path ```python is_super_path(path1, path2) ``` Whether `path1` is the super path of `path2`. Note that if `path1` is same as `path2`, it's also regarded as the super path os `path2`. For instance "/", "/opt" and "/opt/test" are all the super paths of "/opt/test", while "/opt/t" is the super path of "/opt/test". ```python >>> from pydu.path import is_super_path >>> is_super_path('/aa/bb/cc', '/aa/bb/cc') True >>> is_super_path('/aa/bb', '/aa/bb/cc') True >>> is_super_path('/aa/b', '/aa/bb/cc') False ``` ## path.normjoin ```python normjoin(path) ``` Join one or more path components intelligently and normalize it. ```python >>> from pydu.path import normjoin >>> normjoin('/a', '../b') '/b' ``` ## path.filename ```python filename(path) ``` Return the filename without extension. ```python >>> from pydu.path import filename >>> filename('/foo/bar.ext') 'bar' ``` ## path.fileext ```python fileext(path) ``` Return the file extension. If file has not extension, return empty string. ```python >>> from pydu.path import fileext >>> filename('/foo/bar.ext') '.ext' ``` ================================================ FILE: docs/platform.md ================================================ # platform Constants which indicates specific platform. ## platform.WINDOWS Judge whether current platform is WINDOWS or not. ## platform.LINUX Judge whether current platform is LINUX or not. ## platform.POSIX Judge whether current platform is POSIX or not. ## platform.DARWIN Judge whether current platform is DARWIN or not. ## platform.SUNOS Judge whether current platform is SUNOS or not. ## platform.SMARTOS Judge whether current platform is SMARTOS or not. ## platform.FREEBSD Judge whether current platform is FREEBSD or not. ## platform.NETBSD Judge whether current platform is NETBSD or not. ## platform.OPENBSD Judge whether current platform is OPENBSD or not. ## platform.AIX Judge whether current platform is AIX or not. ================================================ FILE: docs/process.md ================================================ # process Utils for handling processes. `process` is based on `psutil`. Need to `pip install psutil` first. ## process.get_processes_by_path ```python get_processes_by_path(path) ``` Get processes which are running on given path or sub path of given path. ```python >>> from pydu.process import get_processes_by_path >>> get_processes_by_path('/usr/bin/python') [{'cmdline': '/usr/bin/python2.7', 'pid': 23383, 'name': 'python'}] ``` ================================================ FILE: docs/request.md ================================================ # Request Utils for handling request. ## request.Filename Supply several methods to get filename. ```python Filename.from_url(url) ``` Detected filename as unicode or None. ```python Filename.from_headers(headers) ``` Detect filename from Content-Disposition headers if present. `headers` could be a dict, list or string. ```python Filename.from_any(dst=None, headers=None, url=None) ``` Detect filename from dst or headers or url. ## request.download ```python Filename.download(url, dst=None) ``` High level function, which downloads URL into tmp file in current directory and then renames it to filename autodetected from either URL or HTTP headers. `url` indicates which url to download. `dst` is the filename or directory of destination. `None` as default, means download to current directory. ## request.check_connect ```python check_connect(ip, port, retry=1, timeout=0.5) ``` Check whether given `ip` and `port` could connect or not. It will `retry` and `timeout` on given. ```python >>> from pydu.request import check_connect >>> check_connect('http://www.baidu.com', 80) '192.168.3.8' ``` ## request.update_query_params ```python update_query_params(url, params) ``` Update query params of given url and return new url. ```python >>> from pydu.request import update_query_params >>> update_query_params('http://example.com', {'foo': 1}) 'http://example.com?foo=1' ``` ## request.cookies_str_to_dict ```python cookies_str_to_dict(cookies) ``` Convert cookies from str to dict. ```python >>> from pydu.request import cookies_str_to_dict >>> cookies_str_to_dict('a=a;b=b') {'a': 'a', 'b': 'b'} ``` ================================================ FILE: docs/set.md ================================================ # Set Additional powerful sets. ## set.OrderedSet ```python OrderedSet(iterable=None) ``` A set which keeps the ordering of the inserted items. ```python >>> from pydu.set import OrderedSet >>> s = OrderedSet([1, 3, 1, 2]) >>> list(s) [1, 3, 2] >>> s.discard(3) >>> list(s) [1, 2] ``` ================================================ FILE: docs/slot.md ================================================ # slot ## slot.SlotBase ```python SlotBase(*args, **kwargs) ``` Base class for class using `__slots__`. If some args or kwargs are not given when initialize class, the value of them will be set with `None`. ```python >>> from pydu.slot import SlotBase >>> class Foo(SlotBase): __slots__ = ('a', 'b', 'c') >>> foo = Foo(1, b=2) >>> foo.a 1 >>> foo.b 2 >>> foo.c >>> ``` ================================================ FILE: docs/string.md ================================================ # String Utils for handling string. ## string.safeunicode ```python safeunicode(obj, encoding='utf-8') ``` Converts any given object to unicode string. ```python >>> from pydu.string import safeunicode >>> safeunicode('hello') u'hello' >>> safeunicode(2) u'2' >>> safeunicode('\xe4\xb8\xad\xe6\x96\x87') u'中文' ``` ## string.safeencode ```python safeencode(obj, encoding='utf-8') ``` Converts any given object to encoded string (default: utf-8). ```python >>> from pydu.string import safeencode >>> safeencode('hello') 'hello' >>> safeencode(2) '2' >>> safeencode(u'中文') '\xe4\xb8\xad\xe6\x96\x87' ``` ## string.lstrips ```python lstrips(text, remove) ``` Removes the string `remove` from the left of `text`. ```python >>> from pydu.string import lstrips >>> lstrips('foobar', 'foo') 'bar' >>> lstrips('FOOBARBAZ', ['FOO', 'BAR']) 'BAZ' >>> lstrips('FOOBARBAZ', ['BAR', 'FOO']) 'BARBAZ' ``` ## string.rstrips ```python rstrips(text, remove) ``` Removes the string `remove` from the right of `text`. ```python >>> from pydu.string import rstrips >>> rstrips('foobar', 'bar') 'foo' ``` ## string.strips ```python strips(text, remove) ``` Removes the string `remove` from the both sides of `text`. ```python >>> from pydu.string import strips >>> strips('foobarfoo', 'foo') 'bar' ``` ## string.common_prefix ```python common_prefix(l) ``` Return common prefix of the stings ```python >>> from pydu.string import common_prefix >>> common_prefix(['abcd', 'abc1']) 'abc' ``` ## string.common_suffix ```python common_suffix(l) ``` Return common suffix of the stings ```python >>> from pydu.string import common_suffix >>> common_suffix(['dabc', '1abc']) 'abc' ``` ## string.sort ```python sort(s, reversed=False) ``` Sort given string by ascending order. If `reverse` is `True`, sorting given string by descending order. ```python >>> from pydu.string import sort >>> sort('dabc') 'abcd' ``` ================================================ FILE: docs/system.md ================================================ # System Utils for handling system, like to track file, make directory, link and so on. ## system.FileTracker ```python FileTracker() ``` Track current opening files, started with `FileTracker.track()`. When opening several files, `FileTracker` tracks them and you can locate them by calling `FileTraker.get_openfiles()`. ```python FiltTracker.track() ``` Start tracking opening files. ```python FiltTracker.untrack() ``` Stop tracking opening files. ```python FiltTracker.get_openfiles() ``` Get current opening files. ```python >>> from pydu.system import FileTracker >>> FileTracker.track() >>> f = open('test', 'w') >>> FileTracker.get_openfiles() {<_io.TextIOWrapper name='test' mode='w' encoding='UTF-8'>} >>> f.close() >>> FileTracker.get_openfiles() set() >>> FileTracker.untrack() >>> f = open('test', 'w') >>> FileTracker.get_openfiles() set() ``` ## system.makedirs ```python makedirs(path, mode=0o755, ignore_errors=False, exist_ok=False) ``` Based on `os.makedirs`,create a leaf directory and all intermediate ones. `mode` default is `0o755`. When make an exists path, if exist_ok is false, `makedirs` will raise an `Exception`. If `ignore_errors` which will ignore all errors raised by `os.makedirs`. ```python >>> from pydu.system import makedirs >>> makedirs('test1/test2') >>> makedirs('test1',exist_ok=True) >>> makedirs('test1') Traceback (most recent call last): ... OSError: Create dir: test1 error. ``` ## system.remove ```python remove(path, mode=0o755, ignore_errors=False, onerror) ``` Remove a file or directory. If `ignore_errors` is set, errors are ignored; otherwise, if `onerror` is set, it is called to handle the error with arguments (`func` , `path` , `exc_info` ) where func is platform and implementation dependent; `path` is the argument to that function that caused it to fail; and `exc_info` is a tuple returned by `sys.exc_info()`. If `ignore_errors` is `False` and `onerror` is None, it attempts to set `path` as writeable and then proceed with deletion if `path` is read-only, or raise an exception if `path` is not read-only. ```python >>> from pydu.system import makedirs >>> from pydu.system import remove >>> from pydu.system import touch >>> makedirs('test') >>> remove('test') >>> touch('test.txt') >>> remove('test.txt') >>> remove('test.txt', ignore_errors=True) >>> remove('test.txt') Traceback (most recent call last): ... OSError: Remove path: test error. Reason: [Errno 2] No such file or directory: 'test.txt' ``` ## system.removes ```python removes(paths, mode=0o755, ignore_errors=False, onerror) ``` Remove a list of file and/or directory.Other parameters same as `remove`. ```python >>> from pydu.system import makedirs >>> from pydu.system import remove >>> from pydu.system import open_file >>> makedirs('test1') >>> makedirs('test2') >>> open_file('test.txt') >>> removes(['test.txt','test1','test2']) ``` ## system.open_file ```python open_file(path, mode='wb+', buffer_size=-1, ignore_errors=False): ``` Open a file, defualt mode `wb+`. If path not exists, it will be created automatically. If `ignore_errors` is set, errors are ignored. ```python >>> from pydu.system import open_file >>> open_file('test.txt') >>> ls test.txt >>> open_file('test1.txt',mode='r') Traceback (most recent call last): ... OSError: Open file: test1.txt error ``` ## system.copy ```python copy(src, dst, ignore_errors=False, follow_symlinks=True): ``` Copy data and mode bits (`cp src dst`).Both the source and destination may be a directory.When `copy` a directory,which contains a symlink,if the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic links are copied.When copy a file,if follow_symlinks is false and src is a symbolic link, a new symlink will be created instead of copying the file it points to,else the contents of the file pointed to by symbolic links is copied. ```python >>> from pydu.system import copy,symlink >>> from pydu.system import makedirs,open_fle >>> open_fle('test/test.txt') >>> symlink('test/test.txt','test/test.link') >>> copy('test/test.link','test/test_copy1.link') >>> copy('test/test.link','test/test_copy2.link',follow_symlink=False) ``` ## system.touch ```python touch(path): ``` Make a new file. ```python >>> from pydu.system import touch >>> touch('test.txt') ``` ## system.symlink ```python symlink(src, dst, overwrite=False, ignore_errors=False) ``` It create a symbolic link pointing to source named link_name.If dist is exist and overwrite is true,a new symlink will be created. ```python >>> from pydu.system import symlink >>> symlink('test.txt','test.link') ``` !> `symlink` can only be used on `unix-like` system. ## system.link ```python link(src, dst, overwrite=False, ignore_errors=False): ``` It create a hard link pointing to source named link_name.If dist is exist and overwrite is true, a new link will be created. ```python >>> from pydu.system import link >>> link('test.txt','test.link') ``` !> `link` can only be used on `unix-like` system. ## system.which ```python which(cmd, mode=os.F_OK | os.X_OK, path=None): ``` Given a command, mode, and a PATH string, return the path which conforms to the given mode on the PATH, or None if there is no such file. `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result of os.environ.get("PATH"), or can be overridden with a custom search path. `which` is `shutil.which` in Python 3. ```python >>> from pydu.system import which >>> which('echo') /bin/echo ``` ## system.chmod ```python chmod(path, mode, recursive=False) ``` Change permissions to the given mode. If `recursive` is True perform recursively. ```python >>> from pydu.system import chmod >>> chmod('/opt/sometest', 0o744) >>> oct(os.stat('/opt/sometest').st_mode)[-3:] '744' ``` !> Although Windows supports `chmod`, you can only set the file’s read-only flag with it (via the stat.S_IWRITE and stat.S_IREAD constants or a corresponding integer value). All other bits are ignored. ## system.chcp ```python chcp(code) ``` Context manager which sets the active code page number. It could also be used as function. ```python >>> from pydu.system import chcp >>> chcp(437) >>> with chcp(437): ... pass >>> ``` !> `chcp` can only be used on `Windows` system. ## system.preferredencoding ```python preferredencoding(code) ``` Get best encoding for the system. ================================================ FILE: docs/unit.md ================================================ # Unit Utils for handling unit. ## unit.Bytes ```python Bytes(bytes) ``` Supply several methods dealing with bytes. ```python Bytes.convert(self, unit=None, multiple=1024) ``` Convert bytes with given `unit`. If `unit` is `None`, convert bytes with suitable unit. Convert `multiple` is default to be 1024. ```python >>> Bytes(1024).convert() (1, 'KB') ``` ================================================ FILE: docs/zh-cn/README.md ================================================ ## pydu > **pydu(Python Data structures and Utils)** 是面向Python 2 和 3 的实用数据结构和工具库。 它收集自开源项目，也有来自开发者贡献。 ## 安装 要安装**pydu**，简单执行： ```bash $ pip install pydu ``` ================================================ FILE: docs/zh-cn/_sidebar.md ================================================ * 模块 * [Archive 归档](zh-cn/archive.md) * [Commad 命令](zh-cn/cmd.md) * [Compat 兼容性](zh-cn/compat.md) * [Console 控制台](zh-cn/console.md) * [Convert 转换](zh-cn/convert.md) * [Dict 字典](zh-cn/dict.md) * [Date and Time 时间](zh-cn/dt.md) * [Environment 环境](zh-cn/environ.md) * [Exception 异常](zh-cn/exception.md) * [Functional 函数式](zh-cn/functional.md) * [Inspect 检查](zh-cn/inspect.md) * [Iter 迭代](zh-cn/iter.md) * [List 列表](zh-cn/list.md) * [Miscellanea 综合](zh-cn/misc.md) * [Network 网络](zh-cn/network.md) * [Path 路径](zh-cn/path.md) * [Platform 平台](zh-cn/platform.md) * [Process 进程](zh-cn/process.md) * [Request 请求](zh-cn/request.md) * [Set 集合](zh-cn/set.md) * [Slot 槽](zh-cn/slot.md) * [String 字符串](zh-cn/string.md) * [System 系统](zh-cn/system.md) * [Unit 单位](zh-cn/unit.md) * [变更记录](changelog.md) ================================================ FILE: docs/zh-cn/archive.md ================================================ # archive 提供归档相关工具，如解压。 ## archive.extract ```python extract(path, to_path='', ext='') ``` 解压tar或zip文件，可指定 ``to_path`` 解压到特定目录。它支持很多文件格式，包括 " "``.tar``、``.tar.bz2``、``.tar.gz``、``.tgz``、``.tz2``、``.zip``。如果给定的 " "``path`` 不包含文件格式，则可指定 ``ext`` 参数来说明文件格式。 ```python >>> from pydu.archive import extract >>> extract('foobar.tgz', '/tmp') >>> extract('foobar', '/tmp', ext='.tgz') >>> extract('foobar', '/tmp') Traceback (most recent call last): ... AttributeError: pydu.archive.UnrecognizedArchiveFormat: Path not a recognized archive format: foobar ``` ================================================ FILE: docs/zh-cn/cmd.md ================================================ # cmd 提供运行命令和获取命令行等工具。 ## cmd.TimeoutExpired ```python TimeoutExpired(cmd, timeout, output=None, stderr=None) ``` 该异常在等待子进程超时时抛出。 属性： cmd, output, stdout, stderr, timeout ## cmd.run ```python run(cmd, shell=False, env=None, timeout=None, timeinterval=1) ``` Run 命令基于 `subprocess.Popen` ，并返回 `(returncode, stdout)` 的这样元组。 注意，`stderr` 被重定向到了 `stdout`。`shell` 同 `Popen` 中的参数一样。 如果在 `timeout` 秒后进程没有退出，将会抛出 `TimeoutExpired` 异常。`timeinterval` 在Python 2中给定 timeout时生效。它表示进程状态检查时间间隔。 如果超时了，子进程不会被杀掉。为了合理清除表现良好的应用，应该要杀掉子进程，并且结束通信。 ```python >>> from pydu.cmd import run >>> run('echo hello') (0, b'hello\r\n') # Python 3 ``` ## cmd.run_with_en_env ```python run_with_en_env(cmd, shell=False, env=None, timeout=None, timeinterval=1) ``` 在英文字符集环境下运行命令，从而得到英文输出。参数同 `run`。 ## cmd.terminate ```python terminate(pid) ``` 根据给定 `pid` 终止进程。 在Windows上，使用 `kernel32.TerminateProcess` 来终止。在其他平台上，使用携带 `signal.SIGTERM` 信号的 `os.kill` 来终止。 ## cmd.cmdline_argv ```python cmdline_argv() ``` 获取当前Python进程的命令行参数。在Windows上使用Python 2时， `cmdline_argv` 的 实现是基于 `shell32.GetCommandLineArgvW` 获取列表元素为Unicode字符串形式的`sys.argv`。 而在其他平台或者是使用Python 3时， `cmdline_argv` 和 `sys.argv` 相同。 ```python >>> from pydu.cmd import cmdline_argv >>> cmdline_argv() ['/Applications/PyCharm.app/Contents/helpers/pydev/pydevconsole.py', '61253', '61254'] ``` ================================================ FILE: docs/zh-cn/compat.md ================================================ # compat 提供Python 2和3兼容的数据结构、库和函数。 ## compat.PY2 判断当前Python解释器是Python 2还是3。 ## compat.urlib ```python urlib(base, url, allow_fragments=True) ``` 在PY2中是 ``urllib``，在PY3中是 ``urllib.request``。 ## compat.urlparse ```python urlparse(base, url, allow_fragments=True) ``` 在PY2中是 ``urlparse``，在PY3中是 ``urllib.parse``。 ## compat.urljoin ```python urljoin(base, url, allow_fragments=True) ``` 在PY2中是 ``urlparse.urljoin``，在PY3中是 ``urllib.parse.urljoin``。 ## compat.iterkeys ```python iterkeys(d) ``` 返回字典键的iter对象。 ## compat.itervalues ```python itervalues(d) ``` 返回字典值的iter对象。 ## compat.iteritems ```python iteritems(d) ``` 返回字典键值对的iter对象。 ## compat.text_type text类型在PY2中是 ``unicode``，在PY3中是 ``str``。 ## compat.string_types string类型在PY2中是 ``(str, unicode)``，在PY3中是 ``(str,)``。 ## compat.strbytes_types strbytes（string bytes）类型在PY2中是 ``(str, unicode, bytes)``，在PY3中是 ``(str, " "bytes)``。 ## compat.numeric_types 在PY2中是 ``(int, long)``，在PY3中是 ``(int,)``。 ## compat.imap ```python imap(func, *iterables) ``` 在PY2中是 ``itertools.imap``，在PY3中是 ``map``。 ## compat.izip ```python izip(iter1 [,iter2 [...]) ``` 在PY2中是 ``itertools.izip``，在PY3中是 ``zip``。 ## compat.reduce ```python reduce(function, sequence, initial=None) ``` 在PY2中是内建 ``reduce``，在PY3中是 ``functools.reduce``。 ## compat.cmp ```python cmp(x, y) ``` Same to ``cmp`` on PY2, but implement on PY3. 在PY2中是内建 ``cmp``，在PY3中则由pydu实现。 ## compat.has_next_attr ```python has_next_attr(x) ``` 查看是否有next属性。 ## compat.is_iterable ```python is_iterable(x) ``` 查看是否是可迭代的。 ```python >>> from pydu.compat import is_iterable >>> is_iterable([]) True >>> is_iterable(1) False ``` ================================================ FILE: docs/zh-cn/console.md ================================================ # Console 提供处理控制台的工具。 ## console.console_size ```python console_size(fallback=(80, 25)) ``` 对于Windows系统，返回可用窗口区域的(width, height)。如果没有控制台，则返回fallback。 对于POSIX系统，返回控制终端的(width, height)。如果遇到IOError，比如没有控制台，返回fallback。 对于其他系统，返回fallback。Fallback默认为(80, 25)，这是大多数终端模拟器的默认大小。 ```python >>> from pydu.console import console_size >>> console_size() (80, 25) ``` ================================================ FILE: docs/zh-cn/convert.md ================================================ # Convert 提供将一类数据转换为另一类的工具。 ## convert.boolean ```python boolean(obj) ``` 将对象转换为布尔值。 如果对象是字符串，将会以不区分大小写的形式转换： * 将 `yes`、 `y`、 `on`、 `true`、 `t`、 `1` 转换为True * 将 `no`、 `n`、 `off`、 `false`、 `f`、 `0` 转换为False * 如果传入其他值，抛出TypeError 如果对象不是字符串，将会使用 `bool(obj)` 转换。 ```python >>> from pydu.string import boolean >>> boolean('yes') True >>> boolean('no') False ``` ## convert.bin2oct ```python bin2oct(x) ``` 把二进制字符串转换为八进制字符串。 比如：'1001' -> '11' ```python >>> from pydu.convert import bin2oct >>> bin2oct('1001') '11' ``` ## convert.bin2dec ```python bin2dec(x) ``` 把二进制字符串转换为十进制数字。 比如：'11' -> 3 ```python >>> from pydu.convert import bin2dec >>> bin2dec('11') 3 ``` ## convert.bin2hex ```python bin2hex(x) ``` 把二进制字符串转换为十六进制字符串。 比如：'11010' -> '1a' ```python >>> from pydu.convert import bin2hex >>> bin2hex('11010') '1a' ``` ## convert.oct2bin ```python oct2bin(x) ``` 把八进制字符串转换为二进制字符串。 比如：'11' -> '1001' ```python >>> from pydu.convert import oct2bin >>> oct2bin('11') '1001' ``` ## convert.oct2dec ```python oct2dec(x) ``` 把八进制字符串转换为十进制数字。 比如：'11' -> 9 ```python >>> from pydu.convert import oct2dec >>> oct2dec('11') 9 ``` ## convert.oct2hex ```python oct2hex(x) ``` 把八进制字符串转换为十六进制字符串。 比如：'32' -> '1a' ```python >>> from pydu.convert import oct2hex >>> oct2hex('32') '1a' ``` ## convert.dec2bin ```python dec2bin(x) ``` 把十进制数字转换为二进制字符串。 比如：3 -> '11' ```python >>> from pydu.convert import dec2bin >>> dec2bin(3) '11' ``` ## convert.dec2oct ```python dec2oct(x) ``` 把十进制数字转换为八进制字符串。 比如：9 -> '11' ```python >>> from pydu.convert import dec2oct >>> dec2oct(9) '11' ``` ## convert.dec2hex ```python dec2hex(x) ``` 把十进制数字转换为十六进制字符串。 比如：26 -> '1a' ```python >>> from pydu.convert import dec2hex >>> dec2hex(26) '1a' ``` ## convert.hex2bin ```python hex2bin(x) ``` 把十六进制字符串转换为二进制字符串。 比如：'1a' -> '11010' ```python >>> from pydu.convert import hex2bin >>> hex2bin('1a') '11010' ``` ## convert.hex2oct ```python hex2oct(x) ``` 把十六进制字符串转换为八进制字符串。 比如：'1a' -> '32' ```python >>> from pydu.convert import hex2oct >>> hex2oct('1a') '32' ``` ## convert.hex2dec ```python hex2dec(x) ``` 把十六进制字符串转换为十进制数字。 比如：'1a' -> 26 ```python >>> from pydu.convert import hex2dec >>> hex2dec('1a') 26 ``` ================================================ FILE: docs/zh-cn/dict.md ================================================ # Dict 增强的字典和相关函数。 ## dict.AttrDict ```python AttrDict(seq=None, **kwargs) ``` AttrDict 对象类似于字典，除了能使用 `obj['foo']`，还能使用 `obj.foo`。 ```python >>> from pydu.dict import AttrDict >>> o = AttrDict(a=1) o.a 1 >>> o['a'] 1 >>> o.a = 2 >>> o['a'] 2 >>> del o.a >>> o.a Traceback (most recent call last): ... AttributeError: 'a' ``` ## dict.CaseInsensitiveDict ```python CaseInsensitiveDict(data=None, **kwargs) ``` 大小写不敏感类 `字典` 对象。实现了 `collections.MutableMapping` 的所有方法和操作， 也实现了字典的 `copy`，此外还提供 `lower_items`。所有的键都应是字符串。 内部结构会记住最后一次被设置的键的大小写，`iter(instance)`、`keys()`、`items()`、 `iterkeys()` 和 `iteritems()` 将会包含大小写敏感的键。 ```python >>> from pydu.dict import CaseInsensitiveDict >>> cid = CaseInsensitiveDict() >>> cid['Accept'] = 'application/json' >>> cid['aCCEPT'] == 'application/json' True >>> list(cid) == ['Accept'] True ``` ## dict.LookupDict ```python LookupDict(name=None) ``` 字典查找对象。 ```python >>> from pydu.dict import LookupDict >>> d = LookupDict() >>> d['key'] None >>> d['key'] = 1 >>> d['key'] 1 ``` ## dict.OrderedDefaultDict ```python OrderedDefaultDict(default_factory=None, *args, **kwds) ``` 记住插入顺序且能根据默认工厂提供默认值的字典。 当key不存在（仅限通过 `__getitem__` 中）时，无参数调用默认工厂来产生新值。 `OrderedDefaultDict` 和 `collections.defaultdict` 在比较时是等同的。 所有剩余参数和传入 `defaultdict` 构造器中的相同，包括关键字参数。 ```python >>> from pydu.dict import OrderedDefaultDict >>> d = OrderedDefaultDict(int) >>> d['b'] 0 >>> d['a'] 0 >>> d.keys() odict_keys(['b', 'a']) ``` ## dict.attrify ```python attrify(obj) ``` 将对象属性化为 `AttriDict` 或 包含 `AttriDict` 的列表（如果对象为列表）。 如果对象或对象中的元素不是列表或字典，将会返回其本身。 ```python >>> from pydu.dict import attrify >>> attrd = attrify({ 'a': [1, 2, {'b': 'b'}], 'c': 'c', }) >>> attrd ], 'c': 'c'}> >>> attrd.a 1 >>> attrd.a[2].b b >>> attrd.c c ``` ================================================ FILE: docs/zh-cn/dt.md ================================================ # Date and Time 提供处理日期和时间的工具。 ## dt.timer ```python timer(path) ``` timer可以上下文管理器或装饰器的方式统计一次调用的时间。 如果将 `print_func` 赋值为 `sys.stdout.write`、 `logger.info` 或其他， timer将会打印所花时长。 ```python timeit = timer(print_func=sys.stdout.write) with timeit: foo() @timeit def foo(): pass ``` `timer.elapsed` 包含了 `foo` 所花费的整个时间。 ```python >>> timeit = timer(print_func=sys.stdout.write) >>> with timeit: ... os.getcwd() Spent time: 1.7881393432617188e-05s ``` ================================================ FILE: docs/zh-cn/environ.md ================================================ # Environ 提供处理环境相关内容的工具。 ## environ.environ ```python environ(**kwargs) ``` 更新一个或多个环境变量的上下文管理器。 保存先前的环境变量（如果有），并在退出上下文管理器时还原。 如果给定 variable_name=None，表示从环境变量中临时移除该变量。 ```python >>> from pydu.environ import environ >>> with environ(a='a'): ... print(os.environ['a']) ... a ``` ## environ.path ```python path(append=None, prepend=None, replace=None) ``` 更新PATH环境变量的上下文管理器。可将给定的字符串或字符串列表， 插入在PATH的开头和末尾，也可替换PATH。 ```python >>> import os >>> from pydu.environ import path >>> with path(append='/foo'): ... print(os.environ['PATH']) ... /usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/foo ``` ================================================ FILE: docs/zh-cn/exception.md ================================================ # Exception 提供处理异常的工具。 ## exception.ignore ```python ignore(*exceptions) ``` 忽略给定异常的上下文管理器。 ```python >>> from pydu.exception import ignore >>> with ignore(ValueError, AttributeError): ... int('abc') ... int.no_exists_func() ... >>> ``` ## exception.default_if_except ```python default_if_except(exception_clses, default=None) ``` 捕捉给定异常（可以是一组异常，以元组表示）并返回默认值的异常装饰器。 ```python >>> from pydu.exception import default_if_except >>> @default_if_except(ValueError, default=0) ... def foo(value): ... return int(value) >>> foo('abc') 0 ``` ================================================ FILE: docs/zh-cn/functional.md ================================================ # functional 提供函数式编程的工具。 ## functional.compose ```python compose(*funcs) ``` 组成所有函数。前一个函数必须接受一个参数，该参数为后一个函数的输出值。 最后一个函数可以接受任意位置参数和关键字参数。 `compose(f1, f2, f3)(*x)` 同 `f1(f2(f3(*x)))`。 ```python >>> from pydu.functional import compose >>> def f1(a): ... return a+1 ... >>> def f2(a, b=2): ... return a+b ... >>> compose(f1, f2)(1, b=3) 5 ``` ================================================ FILE: docs/zh-cn/inspect.md ================================================ # inspect 提供函数参数检查的工具。 ## inspect.getargspec ```python getargspec(func) ``` 获得函数参数的名称和默认值。 返回由四个字符串组成的元组：(args, vargs, varkw, defaults)。 `args` 是参数名称的列表（可能包含嵌套列表）。 `varargs` 和 `varkw` 是 * 和 ** 参数的名称，或者为 `None`。 `defaults` 是最后n个参数的默认值组成的元组。 ```python >>> from pydu.inspect import getargspec >>> def f(name, address='home', age=25, *args, **kwargs): ... pass ... >>> getargspect(f) ArgSpec(args=['name', 'address', 'age'], varargs='args', keywords='kwargs', defaults=('home', 25)) ``` ## inspect.get_func_args ```python get_func_args(func) ``` 返回参数名称的列表。诸如 `*args` 和 `*kwargs` 的参数不被包含。 ```python >>> from pydu.inspect import get_func_args >>> def f(name, address='home', age=25, *args, **kwargs): ... pass ... >>> get_func_args(f) ['name', 'address', 'age'] ``` ## inspect.get_func_full_args ```python get_func_full_args(func) ``` 返回(参数名称, 默认值)元组的列表。如果参数没有默认值，则在元组中丢弃。 诸如 `*args` 和 `*kwargs` 的参数也被包含在内。 ```python >>> from pydu.inspect import get_func_full_args >>> def f(name, address='home', age=25, *args, **kwargs): ... pass ... >>> get_func_full_args(f) [('name',), ('address', 'home'), ('age', 25), ('*args',), ('**kwargs',)] ``` ## inspect.func_accepts_kwargs ```python func_accepts_kwargs(func) ``` 检查函数是否接受关键字参数。 ```python >>> from pydu.inspect import func_accepts_kwargs >>> def f(**kwargs): ... pass ... >>> func_accepts_kwargs(f) True ``` ## inspect.func_accepts_var_args ```python func_accepts_var_args(func) ``` 检查函数是否接受位置参数。 ```python >>> from pydu.inspect import func_accepts_var_args >>> def f(*vargs): ... pass ... >>> func_accepts_var_args(f) True ``` ## inspect.func_supports_parameter ```python func_supports_parameter(func) ``` 检查函数是否接受给定参数。 ```python >>> from pydu.inspect import func_supports_parameter >>> def f(name): ... pass ... >>> func_supports_parameter(f, 'name') True >>> func_supports_parameter(f, 'unkown') Fasle ``` ## inspect.func_has_no_args ```python func_has_no_args(func) ``` 检查函数是否接受任意参数。 ```python >>> from pydu.inspect import func_has_no_args >>> def f(): ... pass ... >>> func_has_no_args(f) True ``` ================================================ FILE: docs/zh-cn/iter.md ================================================ # iter 提供处理迭代对象的工具。 ## iter.first ```python first(iterable) ``` 获取可迭代对象的第一个项。 ```python >>> from pydu.iter import first >>> first([1, 2]) 1 ``` ## iter.last ```python last(iterable) ``` 获取可迭代对象的最后一个项。注意，由于逐步迭代到最后一项，这可能会较慢。 ```python >>> from pydu.iter import last >>> last([1, 2]) 2 ``` ## iter.all ```python all(iterable, predicate) ``` 如果给定可迭代对象的所有元素套用判定函数都是True，则返回True。 ```python >>> from pydu.iter import all >>> all([0, 1, 2], lambda x: x+1) True ``` ## iter.any ```python any(iterable) ``` 如果给定可迭代对象的任一元素套用判定函数是True，则返回True。 ```python >>> from pydu.iter import any >>> any([-1, -1, 0], lambda x: x+1) True ``` ## iter.join ```python join(iterable, separator='') ``` 将可迭代对象中的每一项连接为字符串。 ```python >>> from pydu.iter import join >>> join([1, '2', 3], separator=',') '1,2,3' ``` ================================================ FILE: docs/zh-cn/list.md ================================================ # list 提供处理列表的工具。 ## list.uniq ```python uniq(seq, key=None) ``` 从列表中删除重复的元素，同时保留其余的顺序。 可选参数 `key` 的值应该是一个函数，它接受一个参数并返回一个 `key` 来测试唯一性。 ```python >>> from pydu.list import uniq >>> uniq([1, 4, 0, 2, 0, 3]) [1, 4, 0, 2, 3] ``` ## list.tolist ```python tolist(obj) ``` 将给定的 `obj` 转换为列表。 如果 `obj` 不是列表，返回 `[obj]`，否则返回 `obj` 本身。 ```python >>> from pydu.list import tolist >>> tolist('foo') ['foo'] ``` ## list.flatten ```python flatten(seq) ``` 生成给定 `seq` 中的每个元素。如果元素是可迭代的并且不是字符串， 就递归 `yield` 元素中的每个子元素。 ```python >>> from pydu.list import flatten >>> flatten([1, [2, [3, 4]]]) [1, 2, 3, 4] ``` ================================================ FILE: docs/zh-cn/misc.md ================================================ # misc 提供诸如 `timeout`、 `trace` 等综合性工具。 ## misc.timeout ```python timeout(seconds) ``` 该函数装饰任何可能会hang住一段时间的函数。参数 `seconds` 应为整数。 在 `test.py` 中，你可以这么写： ```python import time from pydu.misc import unix_timeout @timeout(1) def f(): time.sleep(1.01) f() ``` 然后运行 `test.py`，将会看到 `TimeoutError`。 ## misc.trace ```python trace(obj) ``` 跟踪运行中程序的每条语句和行号，就像 `bash -x` 。在 `test.py` 中，你可以这么写： ```python from pydu.misc import trace @trace def f(): print(1) a = 1 + 5 b = [a] print(2) f() ``` 然后运行 `test.py`，将会看到如下控制台输出： ```console test.py(4): print(1) 1 test.py(5): a = 1 + 5 test.py(6): b = [a] test.py(7): print(2) 2 ``` ## misc.memoize ```python memoize(obj) ``` 简单的缓存装饰器，可供支持可哈希的位置参数的函数使用。 它还提供 `cache_clear()` 方法来清除缓存。 ```python >>> @memoize ... def foo() ... return 1 ... >>> foo() 1 >>> foo.cache_clear() >>> ``` ## misc.memoize_when_activated ```python memoize_when_activated(obj) ``` 缓存装饰器，默认禁用。它能根据需求启用和禁用。 为效率起见，它只能用于没有参数的类方法。 ```python >>> class Foo: ... @memoize ... def foo() ... print(1) ... >>> f = Foo() >>> # deactivated (default) >>> foo() 1 >>> foo() 1 >>> >>> # activated >>> foo.cache_activate() >>> foo() 1 >>> foo() >>> foo() >>> ``` ## misc.super_len ```python super_len(obj) ``` 获取具有 `__len__` ， `len` ， `fileno` ， `tell` 等属性的对的长度， 比如： `list` ， `tuple` ， `dict`， `file` 等等。 ```python >>> from pydu.misc import super_len >>> super_len([1, 2]) 2 >>> super_len(open('test', 'w')) 0 ``` ================================================ FILE: docs/zh-cn/network.md ================================================ # network 提供处理网络的工具。 ## network.dotted_netmask ```python dotted_netmask(mask) ``` 将mask从 /`xx` 转化为 `xxx.xxx.xxx.xxx` 形式。 `mask` 可以是 `int` 或者 `str`。 ```python >>> from pydu.network import dotted_netmask >>> dotted_netmask('24') '255.255.255.0' >>> dotted_netmask(24) '255.255.255.0' ``` ## network.private_ipv4s ```python private_ipv4s ``` ipv4地址列表。每个项是（ipv4地址，掩码）这样的元组。 ## network.is_ipv4 ```python is_ipv4(ip) ``` 判断给定的 `ip` 是否为 IPV4。 ```python >>> from pydu.network import is_ipv4 >>> is_ipv4('8.8.8.8') True >>> is_ipv4('localhost.localdomain') False ``` ## network.is_ipv6 ```python is_ipv6(ip) ``` 判断给定的 `ip` 是否为 IPV6。 ```python >>> from pydu.network import is_ipv6 >>> is_ipv6('fe80::9e5b:b149:e187:1a18') True >>> is_ipv6('localhost.localdomain') False ``` ## network.get_free_port ```python get_free_port() ``` 获取可以绑定的空闲端口。 ```python >>> from pydu.network import get_free_port >>> get_free_port() 57118 ``` ## network.ip2int ```python ip2int(ip_str) ``` 将IP转换为整数。支持IPV4和IPV6。如果转换失败，将会抛出 `ValueError`。 ```python >>> from pydu.network import ip2int >>> ip2int('10.1.1.1') 167837953 ``` ## network.int2ip ```python int2ip(ip_int) ``` 将整数转换为IP。支持IPV4和IPV6。如果转换失败，将会抛出 `ValueError`。 ```python >>> from pydu.network import int2ip >>> int2ip(167837953) '10.1.1.1' ``` ================================================ FILE: docs/zh-cn/path.md ================================================ # path 提供处理路径的工具。 ## path.cd ```python cd(path) ``` 进入到给定目录的上下文管理器。 ```python >>> from pydu.path import cd >>> with cd('test'): ... pass ``` ## path.is_super_path ```python is_super_path(path1, path2) ``` 判断 `path1` 是否是 `path2` 的父路径（或父父路径等）。 注意如果 `path1` 和 `path2` 一样，它也被视作是 `path2` 的父路径。 比如，\"/\"、\"opt\"或者\"/opt/test\"是\"/opt/test\"的超级父路径， 而\"/opt/t\"则不是。 ```python >>> from pydu.path import is_super_path >>> is_super_path('/aa/bb/cc', '/aa/bb/cc') True >>> is_super_path('/aa/bb', '/aa/bb/cc') True >>> is_super_path('/aa/b', '/aa/bb/cc') False ``` ## path.normjoin ```python normjoin(path) ``` 将一个或多个路径联接，并将之标准化。 ```python >>> from pydu.path import normjoin >>> normjoin('/a', '../b') '/b' ``` ## path.filename ```python filename(path) ``` 返回没有扩展名的文件名。 ```python >>> from pydu.path import filename >>> filename('/foo/bar.ext') 'bar' ``` ## path.fileext ```python fileext(path) ``` 返回文件扩展名。 如果文件没有扩展名，则返回空字符串。 ```python >>> from pydu.path import fileext >>> filename('/foo/bar.ext') '.ext' ``` ================================================ FILE: docs/zh-cn/platform.md ================================================ # platform 表示特定平台的常量。 ## platform.WINDOWS 判断当前操作系统是否为 `WINDOWS` 。 ## platform.LINUX 判断当前操作系统是否为 `LINUX` 。 ## platform.POSIX 判断当前操作系统是否为 `POSIX` 。 ## platform.DARWIN 判断当前操作系统是否为 `DARWIN` 。 ## platform.SUNOS 判断当前操作系统是否为 `SUNOS` 。 ## platform.SMARTOS 判断当前操作系统是否为 `SMARTOS` 。 ## platform.FREEBSD 判断当前操作系统是否为 `FREEBSD` 。 ## platform.NETBSD 判断当前操作系统是否为 `NETBSD` 。 ## platform.OPENBSD 判断当前操作系统是否为 `OPENBSD` 。 ## platform.AIX 判断当前操作系统是否为 `AIX` 。 ================================================ FILE: docs/zh-cn/process.md ================================================ # process 提供处理进程的工具。 `process` 的实现基于 `psutil`。需要先 `pip install psutil`。 ## process.get_processes_by_path ```python get_processes_by_path(path) ``` 获取占用给定路径或者其子路径的进程。 ```python >>> from pydu.process import get_processes_by_path >>> get_processes_by_path('/usr/bin/python') [{'cmdline': '/usr/bin/python2.7', 'pid': 23383, 'name': 'python'}] ``` ================================================ FILE: docs/zh-cn/request.md ================================================ # Request 提供处理请求的工具。 ## request.Filename 提供各类获取文件名的方法。 ```python Filename.from_url(url) ``` 检测文件名为 unicode 或 None。 ```python Filename.from_headers(headers) ``` 从响应头的Content-Disposition（如果有）中获取文件名。 `headers` 可以使字典、列表或者字符串。 ```python Filename.from_any(dst=None, headers=None, url=None) ``` 从目录、响应头部或者路径获取文件名称。 ## request.download ```python Filename.download(url, dst=None) ``` 将URL下载到当前目录的临时文件中，然后重命名为从URL或者HTTP头中自动检测出的文件名。 `url` 是要下载的URL地址。 `dst` 是文件名或目录的目标路径，默认为 `None`，表示下载到当前目录。 ## request.check_connect ```python check_connect(ip, port, retry=1, timeout=0.5) ``` 在给定的 `timeout` 时间内尝试连接给定的 `ip` 和 `port` 。 ```python >>> from pydu.request import check_connect >>> check_connect('http://www.baidu.com', 80) '192.168.3.8' ``` ## request.update_query_params ```python update_query_params(url, params) ``` 更新给定url的查询参数并返回新的url。 ```python >>> from pydu.request import update_query_params >>> update_query_params('http://example.com', {'foo': 1}) 'http://example.com?foo=1' ``` ## request.cookies_str_to_dict ```python cookies_str_to_dict(cookies) ``` 将字符串类型的Cookies转换为字典对象。 ```python >>> from pydu.request import cookies_str_to_dict >>> cookies_str_to_dict('a=a;b=b') {'a': 'a', 'b': 'b'} ``` ================================================ FILE: docs/zh-cn/set.md ================================================ # Set 增强的集合。 ## set.OrderedSet ```python OrderedSet(iterable=None) ``` 保持插入元素有序的集合。 ```python >>> from pydu.set import OrderedSet >>> s = OrderedSet([1, 3, 1, 2]) >>> list(s) [1, 3, 2] >>> s.discard(3) >>> list(s) [1, 2] ``` ================================================ FILE: docs/zh-cn/slot.md ================================================ # slot ## slot.SlotBase ```python SlotBase(*args, **kwargs) ``` 使用 `__slots__` 的类的基类。当初始化类时未给定位置参数或关键字参数。 其值会被设置为 `None`。 ```python >>> from pydu.slot import SlotBase >>> class Foo(SlotBase): __slots__ = ('a', 'b', 'c') >>> foo = Foo(1, b=2) >>> foo.a 1 >>> foo.b 2 >>> foo.c >>> ``` ================================================ FILE: docs/zh-cn/string.md ================================================ # String 提供处理字符串的工具。 ## string.safeunicode ```python safeunicode(obj, encoding='utf-8') ``` 将任何对象转换为 `unicode` 字符串。 ```python >>> from pydu.string import safeunicode >>> safeunicode('hello') u'hello' >>> safeunicode(2) u'2' >>> safeunicode('\xe4\xb8\xad\xe6\x96\x87') u'中文' ``` ## string.safeencode ```python safeencode(obj, encoding='utf-8') ``` 将任何对象转换为编码后字符串（默认为 `utf-8`）。 ```python >>> from pydu.string import safeencode >>> safeencode('hello') 'hello' >>> safeencode(2) '2' >>> safeencode(u'中文') '\xe4\xb8\xad\xe6\x96\x87' ``` ## string.lstrips ```python lstrips(text, remove) ``` 移除字符串 `text` 左侧的 `remove`。 ```python >>> from pydu.string import lstrips >>> lstrips('foobar', 'foo') 'bar' >>> lstrips('FOOBARBAZ', ['FOO', 'BAR']) 'BAZ' >>> lstrips('FOOBARBAZ', ['BAR', 'FOO']) 'BARBAZ' ``` ## string.rstrips ```python rstrips(text, remove) ``` 移除字符串 `text` 右侧的 `remove`。 ```python >>> from pydu.string import rstrips >>> rstrips('foobar', 'bar') 'foo' ``` ## string.strips ```python strips(text, remove) ``` 移除字符串 `text` 两边的 `remove`。 ```python >>> from pydu.string import strips >>> strips('foobarfoo', 'foo') 'bar' ``` ## string.common_prefix ```python common_prefix(l) ``` 返回字符串的共有前缀。 ```python >>> from pydu.string import common_prefix >>> common_prefix(['abcd', 'abc1']) 'abc' ``` ## string.common_suffix ```python common_suffix(l) ``` 返回字符串的共有后缀 ```python >>> from pydu.string import common_suffix >>> common_suffix(['dabc', '1abc']) 'abc' ``` ## string.sort ```python sort(s, reversed=False) ``` 对给定的字符串进行排序，默认是升序，如果 `reverse` 的值为 `True`，将以降序排序。 ```python >>> from pydu.string import sort >>> sort('dabc') 'abcd' ``` ================================================ FILE: docs/zh-cn/system.md ================================================ # System 提供处理系统（如追踪文件、创建目录、链接等）的工具。 ## system.FileTracker ```python FileTracker() ``` 跟踪当前打开的文件，调用 `FileTracker.track()` 开始跟踪。当打开许多文件时，`FileTracker` 能够跟踪它们，你可以通过调用 `FileTracker.get_openfiles()` 来定位得到这些文件对象。 ```python FiltTracker.track() ``` 开始跟踪打开文件。 ```python FiltTracker.untrack() ``` 停止跟踪打开文件。 ```python FiltTracker.get_openfiles() ``` 获取当前已打开的文件。 ```python >>> from pydu.system import FileTracker >>> FileTracker.track() >>> f = open('test', 'w') >>> FileTracker.get_openfiles() {<_io.TextIOWrapper name='test' mode='w' encoding='UTF-8'>} >>> f.close() >>> FileTracker.get_openfiles() set() >>> FileTracker.untrack() >>> f = open('test', 'w') >>> FileTracker.get_openfiles() set() ``` ## system.makedirs ```python makedirs(path, mode=0o755, ignore_errors=False, exist_ok=False) ``` `makedirs` 基于 `os.makedirs` ，它会创建目标文件夹，以及中间文件夹 （当中间文件夹不存在的时候）。 `mode` 默认值为 `0o75`，当被创建的文件夹已经存在的时候， 如果 `eist_ok` 的值为 `False`，`makedirs` 将会抛出异常。 如果 `ignore_errors` 的值为 `True`，所有的异常将会被忽略。 ```python >>> from pydu.system import makedirs >>> makedirs('test1/test2') >>> makedirs('test1',exist_ok=True) >>> makedirs('test1') Traceback (most recent call last): ... OSError: Create dir: test1 error. ``` ## system.remove ```python remove(path, mode=0o755, ignore_errors=False, onerror) ``` 删除文件或者文件夹。 如果 `ignore_errors` 的值为 `True` ，异常将会被忽略； 否者，如果 `onerror` 的值不为 `None` ，那么 `onerror` 函数将会处理这些异常。 `onerror` 的参数包括 `func` ， `path` 和 `exc_info` 。 `path` 表示 `func` 函数处理该路径时抛出的异常； `exc_info` 是由 `sys.exc_info` 返回的元组。 如果 `ignore_errors` 为 `False`，并且 `onerror` 的值为 `None`， 则尝试将只读的 `path` 改为可写并尝试删除，若非只读则抛出异常。 ```python >>> from pydu.system import makedirs >>> from pydu.system import remove >>> from pydu.system import touch >>> makedirs('test') >>> remove('test') >>> touch('test.txt') >>> remove('test.txt') >>> remove('test.txt', ignore_errors=True) >>> remove('test.txt') Traceback (most recent call last): ... OSError: Remove path: test error. Reason: [Errno 2] No such file or directory: 'test.txt' ``` ## system.removes ```python removes(paths, mode=0o755, ignore_errors=False, onerror) ``` 删除多个文件或者（和）文件夹，其他的参数见 `remove`。 ```python >>> from pydu.system import makedirs >>> from pydu.system import remove >>> from pydu.system import open_file >>> makedirs('test1') >>> makedirs('test2') >>> open_file('test.txt') >>> removes(['test.txt','test1','test2']) ``` ## system.open_file ```python open_file(path, mode='wb+', buffer_size=-1, ignore_errors=False): ``` 默认以 `wb+` 的方式打开文件，如果需要被创建的文件的上级目录不存在，该目录将会被创建。如果 `ignore_errors` 为 `True` ，异常将会被忽略。 ```python >>> from pydu.system import open_file >>> open_file('test.txt') >>> ls test.txt >>> open_file('test1.txt',mode='r') Traceback (most recent call last): ... OSError: Open file: test1.txt error ``` ## system.copy ```python copy(src, dst, ignore_errors=False, follow_symlinks=True): ``` 复制源文件（文件夹）到目标文件（文件夹）。当复制的文件夹包含软连接时， 如果 `symlink` 的值为 `True` ，那么在目标文件夹中会创建相应的软连接； 否者将会复制软连接所指向的文件。当复制的文件为软连接的时候， 如果 `symlink` 的值为 `False` ，那么将会创建与软连接指向相同的软连接； 否者，将会复制软连接所指向的文件。 ```python >>> from pydu.system import copy,symlink >>> from pydu.system import makedirs,open_fle >>> open_fle('test/test.txt') >>> symlink('test/test.txt','test/test.link') >>> copy('test/test.link','test/test_copy1.link') >>> copy('test/test.link','test/test_copy2.link',follow_symlink=False) ``` ## system.touch ```python touch(path): ``` 生成一个新的文件。 ```python >>> from pydu.system import touch >>> touch('test.txt') ``` ## system.symlink ```python symlink(src, dst, overwrite=False, ignore_errors=False) ``` 创建指向源文件的软连接。 如果 `overwrite` 的值为 `True` ，那么已存在的软连接将会被覆盖。 ```python >>> from pydu.system import symlink >>> symlink('test.txt','test.link') ``` !> `symlink` 只支持 `Unix类` 的系统。 ## system.link ```python link(src, dst, overwrite=False, ignore_errors=False): ``` 创建指向源文件的硬连接。 如果 `overwrite` 的值为 `True` ，那么已存在的硬连接将会被覆盖。 ```python >>> from pydu.system import link >>> link('test.txt','test.link') ``` !> `link` 只支持 `Unix类` 的系统。 ## system.which ```python which(cmd, mode=os.F_OK | os.X_OK, path=None): ``` 给定命令名称、模式、和环境变量PATH，返回在PATH下符合给定模式的命令的路径， 如果找不到就返回None。 `mode` 默认是 os.F_OK | os.X_OK。 `path` 默认是 os.environ.get("PATH")的结果，也可被被自定义的搜索路径重载。 在Python 3中，`which` 就是 `shutil.which`。 ```python >>> from pydu.system import which >>> which('echo') /bin/echo ``` ## system.chmod ```python chmod(path, mode, recursive=False) ``` 将权限改成给定模式。如果 `recursive` 是True，将会递归。 ```python >>> from pydu.system import chmod >>> chmod('/opt/sometest', 0o744) >>> oct(os.stat('/opt/sometest').st_mode)[-3:] '744' ``` !> 尽管Windows支持 `chmod`，但你只能使用它设置文件的只读标志 （通过 tat.S_IWRITE 和 stat.S_IREAD）常量或者相关整数值。 其他所有位会被忽略。 ## system.chcp ```python chcp(code) ``` 设置活动代码页号的上下文管理器。它也能够被当做函数使用。 ```python >>> from pydu.system import chcp >>> chcp(437) >>> with chcp(437): ... pass >>> ``` !> `chcp` 只能用于 `Windows` 系统。 ## system.preferredencoding ```python preferredencoding(code) ``` 以最佳的方式获取系统编码。 ================================================ FILE: docs/zh-cn/unit.md ================================================ # Unit 提供处理单位的工具。 ## unit.Bytes ```python Bytes(bytes) ``` 提供处理字节的各类方法。 ```python Bytes.convert(self, unit=None, multiple=1024) ``` 将字节转化为给定 `unit`。如果 `unit` 是 `None`，将字节转换为合适的单位。 转换 `multiple` 的默认值是 1024。 ```python >>> Bytes(1024).convert() (1, 'KB') ``` ================================================ FILE: pydu/__init__.py ================================================ """ Useful data structures, utils for Python. """ from __future__ import absolute_import __version__ = '0.7.2' # Set logging handler to avoid "No handler found" warnings. import logging from logging import NullHandler logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) ================================================ FILE: pydu/archive.py ================================================ """ Based on "python-archive" -- http://pypi.python.org/pypi/python-archive/ Copyright (c) 2010 Gary Wilson Jr. and contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ import os import shutil import stat import tarfile import zipfile from . import logger from .compat import string_types class ArchiveException(Exception): """ Base exception class for all archive errors. """ class UnrecognizedArchiveFormat(ArchiveException): """ Error raised when passed file is not a recognized archive format. """ def extract(path, dst='', ext=''): """ Unpack the tar or zip file at the specified path or file to the directory specified by to_path. """ with Archive(path, ext=ext) as archive: archive.extract(dst) class Archive(object): """ The external API class that encapsulates an archive implementation. """ def __init__(self, file, ext=''): """ Arguments: * 'file' can be a string path to a file or a file-like object. * Optional 'ext' argument can be given to override the file-type guess that is normally performed using the file extension of the given 'file'. Should start with a dot, e.g. '.tar.gz'. """ self._archive = self._archive_cls(file, ext=ext)(file) @staticmethod def _archive_cls(file, ext=''): """ Return the proper Archive implementation class, based on the file type. """ if isinstance(file, string_types): filename = file else: try: filename = file.name except AttributeError: raise UnrecognizedArchiveFormat( "File object not a recognized archive format.") lookup_filename = filename + ext base, tail_ext = os.path.splitext(lookup_filename.lower()) cls = extension_map.get(tail_ext) if not cls: base, ext = os.path.splitext(base) cls = extension_map.get(ext) if not cls: raise UnrecognizedArchiveFormat( "Path not a recognized archive format: %s" % filename) return cls def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self.close() def extract(self, dst=''): self._archive.extract(dst) def list(self): self._archive.list() def filenames(self): return self._archive.filenames() def close(self): self._archive.close() class BaseArchive(object): """ Base Archive class. Implementations should inherit this class. """ @staticmethod def _copy_permissions(mode, filename): """ If the file in the archive has some permissions (this assumes a file won't be writable/executable without being readable), apply those permissions to the unarchived file. """ if mode & stat.S_IROTH: os.chmod(filename, mode) def split_leading_dir(self, path): path = str(path) path = path.lstrip('/').lstrip('\\') if '/' in path and (('\\' in path and path.find('/') < path.find( '\\')) or '\\' not in path): return path.split('/', 1) elif '\\' in path: return path.split('\\', 1) else: return path, '' def has_leading_dir(self, paths): """ Returns true if all the paths have the same leading path name (i.e., everything is in one subdirectory in an archive) """ common_prefix = None for path in paths: prefix, rest = self.split_leading_dir(path) if not prefix: return False elif common_prefix is None: common_prefix = prefix elif prefix != common_prefix: return False return True def extract(self, dst): raise NotImplementedError( 'subclasses of BaseArchive must provide an extract() method') def list(self): raise NotImplementedError( 'subclasses of BaseArchive must provide a list() method') def filenames(self): """ Return a list of the filenames contained in the archive. """ raise NotImplementedError() def __del__(self): if hasattr(self, "_archive"): self._archive.close() class TarArchive(BaseArchive): def __init__(self, file): # tarfile's open uses different parameters for file path vs. file obj. if isinstance(file, string_types): self._archive = tarfile.open(name=file) else: self._archive = tarfile.open(fileobj=file) def extract(self, dst): members = self._archive.getmembers() leading = self.has_leading_dir(x.name for x in members) for member in members: name = member.name if leading: name = self.split_leading_dir(name)[1] filename = os.path.join(dst, name) if member.isdir(): if filename and not os.path.exists(filename): os.makedirs(filename) else: try: extracted = self._archive.extractfile(member) except (KeyError, AttributeError) as exc: # Some corrupt tar files seem to produce this # (specifically bad symlinks) logger.error("In the tar file %s the member %s is invalid: %s", name, member.name, exc) else: dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(dirname) with open(filename, 'wb') as outfile: shutil.copyfileobj(extracted, outfile) self._copy_permissions(member.mode, filename) finally: try: extracted.close() except NameError: pass def list(self): self._archive.list() def filenames(self): return self._archive.getnames() def close(self): self._archive.close() class ZipArchive(BaseArchive): def __init__(self, file): # ZipFile's 'file' parameter can be path (string) or file-like obj. self._archive = zipfile.ZipFile(file) def extract(self, dst): namelist = self._archive.namelist() leading = self.has_leading_dir(namelist) for name in namelist: data = self._archive.read(name) info = self._archive.getinfo(name) if leading: name = self.split_leading_dir(name)[1] filename = os.path.join(dst, name) dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(dirname) if filename.endswith(('/', '\\')): # A directory if not os.path.exists(filename): os.makedirs(filename) else: with open(filename, 'wb') as outfile: outfile.write(data) # Convert ZipInfo.external_attr to mode mode = info.external_attr >> 16 self._copy_permissions(mode, filename) def list(self): self._archive.printdir() def filenames(self): return self._archive.namelist() def close(self): self._archive.close() extension_map = { '.tar': TarArchive, '.tar.bz2': TarArchive, '.tar.gz': TarArchive, '.tgz': TarArchive, '.tz2': TarArchive, '.zip': ZipArchive, } ================================================ FILE: pydu/cmd.py ================================================ import os import sys import time import signal import subprocess from subprocess import Popen, PIPE, STDOUT from .platform import WINDOWS from .compat import PY2 if PY2: class TimeoutExpired(Exception): """ This exception is raised when the timeout expires while waiting for a child process. Attributes: cmd, output, stdout, stderr, timeout """ def __init__(self, cmd, timeout, output=None, stderr=None): self.cmd = cmd self.timeout = timeout self.output = output self.stderr = stderr def __str__(self): return ("Command '%s' timed out after %s seconds" % (self.cmd, self.timeout)) @property def stdout(self): return self.output @stdout.setter def stdout(self, value): # There's no obvious reason to set this, but allow it anyway so # .stdout is a transparent alias for .output self.output = value else: TimeoutExpired = subprocess.TimeoutExpired def run(cmd, shell=False, env=None, timeout=None, timeinterval=1): """ Run cmd based on `subprocess.Popen` and return the tuple of `(returncode, stdout)`. Note, `stderr` is redirected to `stdout`. `shell` is same to parameter of `Popen`. If the process does not terminate after `timeout` seconds, a `TimeoutExpired` exception will be raised. `timeinterval` is workable when timeout is given on Python 2. It means process status checking interval. """ p = Popen(cmd, shell=shell, stdout=PIPE, stderr=STDOUT, env=env) if PY2: if timeout: while timeout > 0 and p.poll() is None: timeout = timeout - timeinterval time.sleep(timeinterval) if p.poll() is None: raise TimeoutExpired(cmd, timeout) stdout, _ = p.communicate() return p.poll(), stdout else: stdout, _ = p.communicate(timeout=timeout) return p.poll(), stdout def run_with_en_env(cmd, shell=False, env=None, timeout=None, timeinterval=1): """ Run cmd with English character sets environment, so that the output will be in English. Parameters are same with `run`. """ if WINDOWS: from .system import chcp with chcp(437): return run(cmd, shell=shell, timeout=timeout, timeinterval=timeinterval) else: env = env if env else os.environ.copy() env.update({'LANG': 'en_US.UTF-8'}) return run(cmd, shell=shell, env=env, timeout=timeout, timeinterval=timeinterval) def terminate(pid): """ Terminate process by given pid. On Windows, using Kernel32.TerminateProcess to kill. On Other platforms, using os.kill with signal.SIGTERM to kill. """ if WINDOWS: # http://code.activestate.com/recipes/347462-terminating-a-subprocess-on-windows/ import ctypes PROCESS_TERMINATE = 1 handle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, False, pid) ctypes.windll.kernel32.TerminateProcess(handle, -1) ctypes.windll.kernel32.CloseHandle(handle) else: os.kill(pid, signal.SIGTERM) if PY2 and WINDOWS: # enable passing unicode arguments from command line in Python 2.x # https://stackoverflow.com/questions/846850/read-unicode-characters def cmdline_argv(): """ Uses shell32.GetCommandLineArgvW to get sys.argv as a list of Unicode strings. Versions 2.x of Python don't support Unicode in sys.argv on Windows, with the underlying Windows API instead replacing multi-byte characters with '?'. """ from ctypes import POINTER, byref, cdll, c_int, windll from ctypes.wintypes import LPCWSTR, LPWSTR GetCommandLineW = cdll.kernel32.GetCommandLineW GetCommandLineW.argtypes = [] GetCommandLineW.restype = LPCWSTR CommandLineToArgvW = windll.shell32.CommandLineToArgvW CommandLineToArgvW.argtypes = [LPCWSTR, POINTER(c_int)] CommandLineToArgvW.restype = POINTER(LPWSTR) cmd = GetCommandLineW() argc = c_int(0) raw_argv = CommandLineToArgvW(cmd, byref(argc)) argnum = argc.value sysnum = len(sys.argv) argv = [] if argnum > 0: # Remove Python executable and commands if present start = argnum - sysnum for i in range(start, argnum): argv.append(raw_argv[i]) return argv else: def cmdline_argv(): return sys.argv ================================================ FILE: pydu/compat.py ================================================ """Utilities for make the code run both on Python2 and Python3. """ import sys import types PY2 = sys.version_info[0] == 2 # builtins if PY2: import __builtin__ as builtins else: import builtins # url* if PY2: import urllib as urlib import urlparse from urlparse import urljoin from urllib import urlencode else: import urllib.request as urlib import urllib.parse as urlparse from urllib.parse import urljoin from urllib.parse import urlencode # Dictionary iteration if PY2: iterkeys = lambda d: d.iterkeys() itervalues = lambda d: d.itervalues() iteritems = lambda d: d.iteritems() else: iterkeys = lambda d: iter(d.keys()) itervalues = lambda d: iter(d.values()) iteritems = lambda d: iter(d.items()) # string and text types if PY2: text_type = unicode string_types = (str, unicode) strbytes_types = (str, unicode, bytes) numeric_types = (int, long) else: text_type = str string_types = (str,) strbytes_types = (str, bytes) numeric_types = (int,) # imap, izip if PY2: from itertools import imap, izip else: imap = map izip = zip if PY2: reduce = reduce else: from functools import reduce # next if PY2: has_next_attr = lambda x: x and hasattr(x, 'next') else: has_next_attr = lambda x: x and hasattr(x, '__next__') # Class Type if PY2: ClassTypes = (type, types.ClassType) else: ClassTypes = (type,) # cmp if PY2: cmp = cmp else: cmp = lambda x, y: (x > y) - (x < y) # is iterable def is_iterable(x): """An implementation independent way of checking for iterables.""" try: iter(x) except TypeError: return False else: return True ================================================ FILE: pydu/console.py ================================================ import sys import shutil from .platform import WINDOWS, POSIX if hasattr(shutil, 'get_terminal_size'): # Actually (80, 25) is the default size used by many terminal emulators def console_size(fallback=(80, 25)): return shutil.get_terminal_size(fallback=fallback) else: # http://bitbucket.org/techtonik/python-pager def console_size(fallback=(80, 25)): """ For Windows, return (width, height) of available window area, fallback if no console is allocated. For POSIX system, return (width, height) of console terminal, fallback on IOError, i.e. when no console is allocated. For other system, return fallback. Fallback defaults to (80, 25) which is the default size used by many terminal emulators. """ if WINDOWS: STD_OUTPUT_HANDLE = -11 # get console handle from ctypes import windll, Structure, byref try: from ctypes.wintypes import SHORT, WORD, DWORD except ImportError: from ctypes import ( c_short as SHORT, c_ushort as WORD, c_ulong as DWORD) console_handle = windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE) # CONSOLE_SCREEN_BUFFER_INFO Structure class COORD(Structure): _fields_ = [('X', SHORT), ('Y', SHORT)] class SMALL_RECT(Structure): _fields_ = [('Left', SHORT), ('Top', SHORT), ('Right', SHORT), ('Bottom', SHORT)] class CONSOLE_SCREEN_BUFFER_INFO(Structure): _fields_ = [('dwSize', COORD), ('dwCursorPosition', COORD), ('wAttributes', WORD), ('srWindow', SMALL_RECT), ('dwMaximumWindowSize', DWORD)] sbi = CONSOLE_SCREEN_BUFFER_INFO() ret = windll.kernel32.GetConsoleScreenBufferInfo( console_handle, byref(sbi)) if ret == 0: return fallback return ((sbi.srWindow.Right - sbi.srWindow.Left + 1) or fallback[0], (sbi.srWindow.Bottom - sbi.srWindow.Top + 1) or fallback[1]) elif POSIX: # http://www.kernel.org/doc/man-pages/online/pages/man4/tty_ioctl.4.html from fcntl import ioctl from termios import TIOCGWINSZ from array import array """ struct winsize { unsigned short ws_row; unsigned short ws_col; unsigned short ws_xpixel; /* unused */ unsigned short ws_ypixel; /* unused */ }; """ winsize = array("H", [0] * 4) try: ioctl(sys.stdout.fileno(), TIOCGWINSZ, winsize) except IOError: # for example IOError: [Errno 25] Inappropriate ioctl for device # when output is redirected pass return winsize[1] or fallback[0], winsize[0] or fallback[1] return fallback ================================================ FILE: pydu/convert.py ================================================ import functools from pydu.compat import PY2 def boolean(obj): """ Convert obj to a boolean value. If obj is string, obj will converted by case-insensitive way: * convert `yes`, `y`, `on`, `true`, `t`, `1` to True * convert `no`, `n`, `off`, `false`, `f`, `0` to False * raising TypeError if other values passed If obj is non-string, obj will converted by bool(obj). """ try: text = obj.strip().lower() except AttributeError: return bool(obj) if text in ('yes', 'y', 'on', 'true', 't', '1'): return True elif text in ('no', 'n', 'off', 'false', 'f', '0'): return False else: raise ValueError("Unable to convert {!r} to a boolean value.".format(text)) ########################################################################## # Convert number from one base(2, 8, 10, 16) to another base(2, 8, 10, 16) ########################################################################## _oct_index = 1 if PY2 else 2 def _rstrip_L(func): if PY2: @functools.wraps(func) def wrapper(x): return func(x).rstrip('L') return wrapper return func # binary to octal, decimal and hexadecimal @_rstrip_L def bin2oct(x): """ Convert binary string to octal string. For instance: '1001' -> '11' """ return oct(int(x, 2))[_oct_index:] def bin2dec(x): """ Convert binary string to decimal number. For instance: '11' -> 3 """ return int(x, 2) @_rstrip_L def bin2hex(x): """ Convert binary string to hexadecimal string. For instance: '11010' -> '1a' """ return hex(int(x, 2))[2:] # octal to binary, decimal and hexadecimal @_rstrip_L def oct2bin(x): """ Convert octal string to binary string. For instance: '11' -> '1001' """ return bin(int(x, 8))[2:] def oct2dec(x): """ Convert octal string to decimal number. For instance: '11' -> 9 """ return int(x, 8) @_rstrip_L def oct2hex(x): """ Convert octal string to hexadecimal string. For instance: '32' -> '1a' """ return hex(int(x, 8))[2:] # decimal to binary, octal and hexadecimal @_rstrip_L def dec2bin(x): """ Convert decimal number to binary string. For instance: 3 -> '11' """ return bin(x)[2:] @_rstrip_L def dec2oct(x): """ Convert decimal number to octal string. For instance: 9 -> '11' """ return oct(x)[_oct_index:] @_rstrip_L def dec2hex(x): """ Convert decimal number to hexadecimal string. For instance: 26 -> '1a' """ return hex(x)[2:] # hexadecimal to binary, octal and decimal @_rstrip_L def hex2bin(x): """ Convert hexadecimal string to binary string. For instance: '1a' -> '11010' """ return bin(int(x, 16))[2:] @_rstrip_L def hex2oct(x): """ Convert hexadecimal string to octal string. For instance: '1a' -> '32' """ return oct(int(x, 16))[_oct_index:] def hex2dec(x): """ Convert hexadecimal string to decimal number. For instance: '1a' -> 26 """ return int(x, 16) ================================================ FILE: pydu/dict.py ================================================ # coding: utf-8 import collections try: # Python 3 from collections.abc import Callable, Mapping, MutableMapping except ImportError: # Python 2.7 from collections import Callable, Mapping, MutableMapping from .compat import PY2 class AttrDict(dict): """ A AttrDict object is like a dictionary except `obj.foo` can be used in addition to `obj['foo']`. >>> o = AttrDict(a=1) >>> o.a 1 >>> o['a'] 1 >>> o.a = 2 >>> o['a'] 2 >>> del o.a >>> o.a Traceback (most recent call last): ... AttributeError: 'a' """ def __getattr__(self, key): try: return self[key] except KeyError as k: raise AttributeError(k) def __setattr__(self, key, value): self[key] = value def __delattr__(self, key): try: del self[key] except KeyError as k: raise AttributeError(k) def __repr__(self): return '' class CaseInsensitiveDict(MutableMapping): """ A case-insensitive ``dict``-like object. Implements all methods and operations of ``MutableMapping`` as well as dict's ``copy``. Also provides ``lower_items``. All keys are expected to be strings. The structure remembers the case of the last key to be set, and ``iter(instance)``, ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` will contain case-sensitive keys. However, querying and contains testing is case insensitive: cid = CaseInsensitiveDict() cid['Accept'] = 'application/json' cid['aCCEPT'] == 'application/json' # True list(cid) == ['Accept'] # True For example, ``headers['content-encoding']`` will return the value of a ``'Content-Encoding'`` response header, regardless of how the header name was originally stored. If the constructor, ``.update``, or equality comparison operations are given keys that have equal ``.lower()``s, the behavior is undefined. """ def __init__(self, data=None, **kwargs): self._store = {} if data is None: data = {} self.update(data, **kwargs) def __setitem__(self, key, value): # Use the lowercased key for lookups, but store the actual # key alongside the value. self._store[key.lower()] = (key, value) def __getitem__(self, key): return self._store[key.lower()][1] def __delitem__(self, key): del self._store[key.lower()] def __iter__(self): return (casedkey for casedkey, mappedvalue in self._store.values()) def __len__(self): return len(self._store) def lower_items(self): """Like iteritems(), but with all lowercase keys.""" return ( (lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items() ) def __eq__(self, other): if isinstance(other, Mapping): other = CaseInsensitiveDict(other) else: return NotImplemented # Compare insensitively return dict(self.lower_items()) == dict(other.lower_items()) # Copy is required def copy(self): return CaseInsensitiveDict(self._store.values()) def __repr__(self): return '%s(%r)' % (self.__class__.__name__, dict(self.items())) class LookupDict(dict): """ Dictionary lookup object. d = LookupDict() print(d['key']) # None d['key'] = 1 print(d['key']) # 1 """ def __init__(self, name=None): self.name = name super(LookupDict, self).__init__() def __getitem__(self, key): # We allow fall-through here, so values default to None return self.get(key, None) # https://stackoverflow.com/questions/6190331/can-i-do-an-ordered-default-dict-in-python class OrderedDefaultDict(collections.OrderedDict): """ Dictionary that remembers insertion order and has default value with default factory. The default factory is called without arguments to produce a new value when a key is not present, in `__getitem__` only. An `OrderedDefaultDict` compares equal to a `collections.defaultdict` with the same items. All remaining arguments are treated the same as if they were passed to the `defaultdict` constructor, including keyword arguments. """ def __init__(self, default_factory=None, *args, **kwds): if (default_factory is not None and not isinstance(default_factory, Callable)): raise TypeError('First argument must be callable') super(OrderedDefaultDict, self).__init__(*args, **kwds) self.default_factory = default_factory def __getitem__(self, key): try: return super(OrderedDefaultDict, self).__getitem__(key) except KeyError: return self.__missing__(key) def __missing__(self, key): if self.default_factory is None: raise KeyError(key) self[key] = value = self.default_factory() return value def __reduce__(self): if self.default_factory is None: args = tuple() else: args = self.default_factory, return type(self), args, None, None, self.items() def copy(self): return self.__copy__() def __copy__(self): return self.__class__(self.default_factory, self) if PY2: def __deepcopy__(self, memo): import copy return self.__class__(self.default_factory, copy.deepcopy(self.items())) else: def __deepcopy__(self, memo): import copy return self.__class__(self.default_factory, copy.deepcopy(iter(self.items()))) def __repr__(self): return 'OrderedDefaultDict({default_factory}, {repr})'.format( default_factory=self.default_factory, repr=super(OrderedDefaultDict, self).__repr__() ) def attrify(obj): if isinstance(obj, list): for i, v in enumerate(obj): obj[i] = attrify(v) return obj elif isinstance(obj, dict): attrd = AttrDict() for key, value in obj.items(): value = attrify(value) setattr(attrd, key, value) return attrd else: return obj ================================================ FILE: pydu/dt.py ================================================ import time class timer(object): """ A timer can time how long does calling take as a context manager or decorator. If assign ``print_func`` with ``sys.stdout.write``, ``logger.info`` and so on, timer will print the spent time. """ def __init__(self, print_func=None): self.elapsed = None self.print_func = print_func def __enter__(self): self.start = time.time() def __exit__(self, *_): self.elapsed = time.time() - self.start if self.print_func: self.print_func(self.__str__()) def __call__(self, fun): def wrapper(*args, **kwargs): with self: return fun(*args, **kwargs) return wrapper def __str__(self): return 'Spent time: {}s'.format(self.elapsed) ================================================ FILE: pydu/environ.py ================================================ import os from contextlib import contextmanager from pydu.list import tolist from pydu.compat import iteritems @contextmanager def environ(**kwargs): """ Context manager for updating one or more environment variables. Preserves the previous environment variable (if available) and recovers when exiting the context manager. If given variable_name=None, it means removing the variable from environment temporarily. """ original_kwargs = {} for key in kwargs: original_kwargs[key] = os.environ.get(key, None) if kwargs[key] is None and original_kwargs[key] is not None: del os.environ[key] elif kwargs[key] is not None: os.environ[key] = kwargs[key] yield for key, value in iteritems(original_kwargs): if value is None: os.environ.pop(key, None) continue os.environ[key] = value @contextmanager def path(append=None, prepend=None, replace=None): """ Context manager for updating the PATH environment variable which appends, prepends or replaces the PATH with given string or a list of strings. """ original = os.environ['PATH'] if replace: replace = tolist(replace) os.environ['PATH'] = os.pathsep.join(replace) else: if append: append = tolist(append) append.insert(0, os.environ['PATH']) os.environ['PATH'] = os.pathsep.join(append) if prepend: prepend = tolist(prepend) prepend.append(os.environ['PATH']) os.environ['PATH'] = os.pathsep.join(prepend) yield os.environ['PATH'] = original ================================================ FILE: pydu/exception.py ================================================ import contextlib from functools import wraps from .compat import PY2 if PY2: @contextlib.contextmanager def ignore(*exceptions): try: yield except exceptions: pass else: ignore = contextlib.suppress def default_if_except(exception_clses, default=None): """ A exception decorator which excepts given exceptions and return default value. """ def _default_if_except(func): @wraps(func) def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except exception_clses: return default return wrapper return _default_if_except ================================================ FILE: pydu/functional.py ================================================ from .compat import reduce def compose(*funcs): """ Compose all functions. The previous function must accept one argument, which is the output of the next function. The last function can accept any args and kwargs. compose(f1, f2, f3)(*x) is same to f1(f2(f3(*x))). """ return reduce( lambda f1, f2: (lambda *args, **kwargs: f2(f1(*args, **kwargs))), reversed(funcs)) ================================================ FILE: pydu/inspect.py ================================================ from __future__ import absolute_import import inspect from .compat import PY2 def getargspec(func): """ Get the names and default values of a function's parameters. A tuple of four things is returned: (args, varargs, keywords, defaults). 'args' is a list of the argument names, including keyword-only argument names. 'varargs' and 'keywords' are the names of the * and ** parameters or None. 'defaults' is an n-tuple of the default values of the last n parameters. """ if PY2: return inspect.getargspec(func) else: sig = inspect.signature(func) args = [ p.name for p in sig.parameters.values() if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD ] varargs = [ p.name for p in sig.parameters.values() if p.kind == inspect.Parameter.VAR_POSITIONAL ] varargs = varargs[0] if varargs else None varkw = [ p.name for p in sig.parameters.values() if p.kind == inspect.Parameter.VAR_KEYWORD ] varkw = varkw[0] if varkw else None defaults = [ p.default for p in sig.parameters.values() if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD and p.default is not p.empty ] or None return args, varargs, varkw, defaults def get_func_args(func): """ Return a list of the argument names. Arguments such as *args and **kwargs are not included. """ if PY2: argspec = inspect.getargspec(func) if inspect.ismethod(func): return argspec.args[1:] # ignore 'self' return argspec.args else: sig = inspect.signature(func) return [ name for name, param in sig.parameters.items() if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD and name != 'self' ] def get_func_full_args(func): """ Return a list of (argument name, default value) tuples. If the argument does not have a default value, omit it in the tuple. Arguments such as *args and **kwargs are also included. """ if PY2: argspec = inspect.getargspec(func) if inspect.ismethod(func): args = argspec.args[1:] # ignore 'self' else: args = argspec.args defaults = argspec.defaults or [] # Split args into two lists depending on whether they have default value no_default = args[:len(args) - len(defaults)] with_default = args[len(args) - len(defaults):] # Join the two lists and combine it with default values args = [(arg,) for arg in no_default] + zip(with_default, defaults) # Add possible *args and **kwargs and prepend them with '*' or '**' varargs = [('*' + argspec.varargs,)] if argspec.varargs else [] kwargs = [('**' + argspec.keywords,)] if argspec.keywords else [] return args + varargs + kwargs else: sig = inspect.signature(func) args = [] for arg_name, param in sig.parameters.items(): name = arg_name # Ignore 'self' if name == 'self': continue if param.kind == inspect.Parameter.VAR_POSITIONAL: name = '*' + name elif param.kind == inspect.Parameter.VAR_KEYWORD: name = '**' + name if param.default != inspect.Parameter.empty: args.append((name, param.default)) else: args.append((name,)) return args def func_accepts_kwargs(func): """ Check whether or not the func accepts kwargs. """ # Not all callables are inspectable with getargspec, so we'll # try a couple different ways but in the end fall back on assuming # it is -- we don't want to prevent registration of valid but weird # callables. if PY2: try: argspec = inspect.getargspec(func) except TypeError: try: argspec = inspect.getargspec(func.__call__) except (TypeError, AttributeError): argspec = None return not argspec or argspec[2] is not None else: return any( p for p in inspect.signature(func).parameters.values() if p.kind == p.VAR_KEYWORD ) def func_accepts_var_args(func): """ Check whether or not the func accepts var args. """ if PY2: return inspect.getargspec(func)[1] is not None else: return any( p for p in inspect.signature(func).parameters.values() if p.kind == p.VAR_POSITIONAL ) def func_supports_parameter(func, parameter): """ Check whether or the func supports the given parameter. """ if PY2: args, varargs, varkw, defaults = inspect.getargspec(func) if inspect.ismethod(func): args = args[1:] # ignore 'self' return parameter in args + [varargs, varkw] else: parameters = [name for name in inspect.signature(func).parameters if name != 'self'] return parameter in parameters def func_has_no_args(func): """ Check whether or not the func has any args. """ args = inspect.getargspec(func)[0] if PY2 else [ p for name, p in inspect.signature(func).parameters.items() if p.kind == p.POSITIONAL_OR_KEYWORD and name != 'self' ] return len(args) == 1 ================================================ FILE: pydu/iter.py ================================================ """iteration tools""" from .compat import builtins, imap def first(iterable): """ Get the first item in the iterable. """ return next(iter(iterable)) def last(iterable): """ Get the last item in the iterable. Warning, this can be slow due to iter step by step to last one. """ item = None for item in iterable: pass return item def all(iterable, predicate): """ Returns True if all elements in the given iterable are True for the given predicate function. """ return builtins.all(predicate(x) for x in iterable) def any(iterable, predicate): """ Returns True if any element in the given iterable is True for the given predicate function. """ return builtins.any(predicate(x) for x in iterable) def join(iterable, separator=''): """ Join each item of iterable to string. """ return separator.join(imap(str, iterable)) ================================================ FILE: pydu/list.py ================================================ try: # Python 3 from collections.abc import Iterable except ImportError: # Python 2.7 from collections import Iterable from pydu.compat import strbytes_types def uniq(seq, key=None): """ Removes duplicate elements from a list while preserving the order of the rest. The value of the optional `key` parameter should be a function that takes a single argument and returns a key to test the uniqueness. """ key = key or (lambda x: x) seen = set() uniq_list = [] for value in seq: uniq_value = key(value) if uniq_value in seen: continue seen.add(uniq_value) uniq_list.append(value) return uniq_list def tolist(obj): """ Convert given `obj` to list. If `obj` is not a list, return `[obj]`, else return `obj` itself. """ if not isinstance(obj, list): return [obj] return obj # https://stackoverflow.com/questions/2158395/flatten-an-irregular-list-of-lists def flatten(seq): """ Generate each element of the given `seq`. If the element is iterable and is not string, it yields each sub-element of the element recursively. """ for element in seq: if isinstance(element, Iterable) and \ not isinstance(element, strbytes_types): for sub in flatten(element): yield sub else: yield element ================================================ FILE: pydu/misc.py ================================================ import os import sys import linecache import functools import io from threading import Thread from . import logger class TimeoutError(Exception): pass def timeout(seconds, error_message='Time out'): def decorated(func): @functools.wraps(func) def wrapper(*args, **kwargs): share = [TimeoutError(error_message)] def func_with_except(): try: share[0] = func(*args, **kwargs) except Exception as e: share[0] = e t = Thread(target=func_with_except) t.daemon = True try: t.start() t.join(seconds) except Exception as e: logger.error('Starting timeout thread for %s error', e) raise e result = share[0] if isinstance(result, BaseException): raise result return result return wrapper return decorated def trace(func): # pragma: no cover def globaltrace(frame, why, arg): if why == 'call': return localtrace return None def localtrace(frame, why, arg): if why == 'line': # record the file name and line number of every trace filename = frame.f_code.co_filename lineno = frame.f_lineno bname = os.path.basename(filename) print('{}({}): {}\n'.format( bname, lineno, linecache.getline(filename, lineno).strip('\r\n'))) return localtrace def _func(*args, **kwds): try: sys.settrace(globaltrace) result = func(*args, **kwds) return result finally: sys.settrace(None) return _func # https://github.com/giampaolo/psutil/blob/master/psutil/_common.py def memoize(func): """ A simple memoize decorator for functions supporting (hashable) positional arguments. It also provides a cache_clear() function for clearing the cache: >>> @memoize ... def foo() ... return 1 ... >>> foo() 1 >>> foo.cache_clear() >>> """ @functools.wraps(func) def wrapper(*args, **kwargs): key = (args, frozenset(sorted(kwargs.items()))) try: return cache[key] except KeyError: ret = cache[key] = func(*args, **kwargs) return ret def cache_clear(): """Clear cache.""" cache.clear() cache = {} wrapper.cache_clear = cache_clear return wrapper # https://github.com/giampaolo/psutil/blob/master/psutil/_common.py def memoize_when_activated(func): """ A memoize decorator which is disabled by default. It can be activated and deactivated on request. For efficiency reasons it can be used only against class methods accepting no arguments. >>> class Foo: ... @memoize ... def foo(self) ... print(1) ... >>> f = Foo() >>> # deactivated (default) >>> foo() 1 >>> foo() 1 >>> >>> # activated >>> foo.cache_activate() >>> foo() 1 >>> foo() >>> foo() >>> """ @functools.wraps(func) def wrapper(self): if not wrapper.cache_activated: return func(self) else: try: ret = cache[func] except KeyError: ret = cache[func] = func(self) return ret def cache_activate(): """Activate cache.""" wrapper.cache_activated = True def cache_deactivate(): """Deactivate and clear cache.""" wrapper.cache_activated = False cache.clear() cache = {} wrapper.cache_activated = False wrapper.cache_activate = cache_activate wrapper.cache_deactivate = cache_deactivate return wrapper # https://github.com/requests/requests/blob/master/requests/utils.py def super_len(obj): total_length = None current_position = 0 if hasattr(obj, '__len__'): total_length = len(obj) elif hasattr(obj, 'len'): total_length = obj.len elif hasattr(obj, 'fileno'): try: fileno = obj.fileno() except io.UnsupportedOperation: pass else: total_length = os.fstat(fileno).st_size if hasattr(obj, 'tell'): try: current_position = obj.tell() except (OSError, IOError): # This can happen in some weird situations, such as when the file # is actually a special file descriptor like stdin. In this # instance, we don't know what the length is, so set it to zero and # let requests chunk it instead. if total_length is not None: current_position = total_length else: if hasattr(obj, 'seek') and total_length is None: # StringIO and BytesIO have seek but no useable fileno try: # seek to end of file obj.seek(0, 2) total_length = obj.tell() # seek back to current position to support # partially read file-like objects obj.seek(current_position or 0) except (OSError, IOError): total_length = 0 if total_length is None: total_length = 0 return max(0, total_length - current_position) ================================================ FILE: pydu/network.py ================================================ import socket import struct import ctypes import binascii from contextlib import closing from .platform import WINDOWS from .string import safeencode, safeunicode from .convert import hex2dec, dec2hex # https://github.com/hickeroar/win_inet_pton/blob/master/win_inet_pton.py if WINDOWS: class _sockaddr(ctypes.Structure): _fields_ = [("sa_family", ctypes.c_short), ("__pad1", ctypes.c_ushort), ("ipv4_addr", ctypes.c_byte * 4), ("ipv6_addr", ctypes.c_byte * 16), ("__pad2", ctypes.c_ulong)] WSAStringToAddressA = ctypes.windll.ws2_32.WSAStringToAddressA WSAAddressToStringA = ctypes.windll.ws2_32.WSAAddressToStringA def _win_inet_pton(address_family, ip_str): ip_str = safeencode(ip_str) addr = _sockaddr() addr.sa_family = address_family addr_size = ctypes.c_int(ctypes.sizeof(addr)) if WSAStringToAddressA( ip_str, address_family, None, ctypes.byref(addr), ctypes.byref(addr_size) ) != 0: raise socket.error(ctypes.FormatError()) if address_family == socket.AF_INET: return ctypes.string_at(addr.ipv4_addr, 4) if address_family == socket.AF_INET6: return ctypes.string_at(addr.ipv6_addr, 16) raise socket.error('unknown address family') def _win_inet_ntop(address_family, packed_ip): addr = _sockaddr() addr.sa_family = address_family addr_size = ctypes.c_int(ctypes.sizeof(addr)) ip_string = ctypes.create_string_buffer(128) ip_string_size = ctypes.c_int(ctypes.sizeof(ip_string)) if address_family == socket.AF_INET: if len(packed_ip) != ctypes.sizeof(addr.ipv4_addr): raise socket.error('packed IP wrong length for inet_ntoa') ctypes.memmove(addr.ipv4_addr, packed_ip, 4) elif address_family == socket.AF_INET6: if len(packed_ip) != ctypes.sizeof(addr.ipv6_addr): raise socket.error('packed IP wrong length for inet_ntoa') ctypes.memmove(addr.ipv6_addr, packed_ip, 16) else: raise socket.error('unknown address family') if WSAAddressToStringA( ctypes.byref(addr), addr_size, None, ip_string, ctypes.byref(ip_string_size) ) != 0: raise socket.error(ctypes.FormatError()) return ip_string[:ip_string_size.value - 1] socket.inet_pton = _win_inet_pton socket.inet_ntop = _win_inet_ntop # https://github.com/kennethreitz/requests/blob/master/requests/utils.py def dotted_netmask(mask): """ Converts mask from /xx format to xxx.xxx.xxx.xxx Example: if mask is 24 function returns 255.255.255.0 """ mask = int(mask) bits = 0xffffffff ^ (1 << 32 - mask) - 1 return socket.inet_ntoa(struct.pack('>I', bits)) # http://en.wikipedia.org/wiki/Private_network private_ipv4s = [ ('10.0.0.0', 8), # 10.0.0.0 - 10.255.255.255 ('172.16.0.0', 12), # 172.16.0.0 - 172.31.255.255 ('192.168.0.0', 16), # 192.168.0.0 - 192.168.255.255 ] # https://github.com/kennethreitz/requests/blob/master/requests/utils.py def is_ipv4(ip): """ Returns True if the IPv4 address ia valid, otherwise returns False. """ try: socket.inet_aton(ip) except socket.error: return False return True def is_ipv6(ip): """ Returns True if the IPv6 address ia valid, otherwise returns False. """ try: socket.inet_pton(socket.AF_INET6, ip) except socket.error: return False return True def get_free_port(): with closing(socket.socket(socket.AF_INET, type=socket.SOCK_STREAM)) as s: s.bind(('127.0.0.1', 0)) _, port = s.getsockname() return port # https://stackoverflow.com/questions/5619685/conversion-from-ip-string-to-integer-and-backward-in-python # https://stackoverflow.com/questions/11894717/python-convert-ipv6-to-an-integer def ip2int(ip_str): """ Convert ip to integer. Support IPV4 and IPV6. Raise `ValueError` if convert failed. """ try: return struct.unpack("!I", socket.inet_aton(ip_str))[0] except socket.error: pass try: return hex2dec(binascii.hexlify(socket.inet_pton(socket.AF_INET6, ip_str))) except socket.error: pass raise ValueError('{!r} does not appear to be an IPv4 or IPv6 address'.format(ip_str)) # https://stackoverflow.com/questions/5619685/conversion-from-ip-string-to-integer-and-backward-in-python def int2ip(ip_int): """ Convert integer to ip. Support IPV4 and IPV6. Raise `ValueError` if convert failed. """ try: return socket.inet_ntoa(struct.pack("!I", ip_int)) except (socket.error, struct.error): pass try: ip_str = socket.inet_ntop(socket.AF_INET6, binascii.unhexlify(dec2hex(ip_int))) return safeunicode(ip_str, encoding='ascii') except (socket.error, struct.error): pass raise ValueError('{!r} does not appear to be an IPv4 or IPv6 address'.format(ip_int)) ================================================ FILE: pydu/path.py ================================================ import os from contextlib import contextmanager @contextmanager def cd(path): """ Context manager for cd the given path. """ cwd = os.getcwd() os.chdir(path) yield os.chdir(cwd) def is_super_path(path1, path2): """ Whether `path1` is the super path of `path2`. Note that if `path1` is same as `path2`, it's also regarded as the super path os `path2`. For instance "/", "/opt" and "/opt/test" are all the super paths of "/opt/test", while "/opt/t" is the super path of "/opt/test". """ path1 = os.path.normpath(path1) current_path2 = os.path.normpath(path2) parent_path2 = os.path.dirname(current_path2) if path1 == current_path2: return True while parent_path2 != current_path2: if path1 == parent_path2: return True current_path2 = parent_path2 parent_path2 = os.path.dirname(parent_path2) return False def normjoin(path, *paths): """Join one or more path components intelligently and normalize it.""" return os.path.normpath(os.path.join(path, *paths)) def filename(path): """Return the filename without extension.""" return os.path.splitext(os.path.basename(path))[0] def fileext(path): """ Return the file extension. If file has not extension, return empty string. """ return os.path.splitext(os.path.basename(path))[1] ================================================ FILE: pydu/platform.py ================================================ import os import sys WINDOWS = os.name == 'nt' LINUX = sys.platform.startswith('linux') POSIX = os.name == 'posix' DARWIN = sys.platform.startswith('darwin') SUNOS = sys.platform.startswith('sunos') SMARTOS = os.uname()[3].startswith('joyent_') if SUNOS else False FREEBSD = sys.platform.startswith('freebsd') NETBSD = sys.platform.startswith('netbsd') OPENBSD = sys.platform.startswith('openbsd') AIX = sys.platform.startswith('aix') ================================================ FILE: pydu/process.py ================================================ try: import psutil except ImportError: raise ImportError('Need to pip install psutil if you use pydu.process') from .path import is_super_path def get_processes_by_path(path): """ Get processes which are running on given path or sub path of given path. """ pinfos = [] for proc in psutil.process_iter(): pinfo = proc.as_dict(attrs=['pid', 'name', 'exe', 'cwd', 'open_files']) using_paths = [] if pinfo['exe']: using_paths.append(pinfo['exe']) if pinfo['cwd']: using_paths.append(pinfo['cwd']) if pinfo['open_files']: using_paths.extend(pinfo['open_files']) for using_path in using_paths: if is_super_path(path, using_path): continue pinfos.append({ 'pid': pinfo['pid'], 'name': pinfo['name'], 'cmdline': pinfo['exe'] }) return pinfos ================================================ FILE: pydu/request.py ================================================ import os import shutil import tempfile import socket from . import logger from .string import safeunicode from .compat import PY2, string_types, urlparse, urlib, urlencode class FileName(object): @staticmethod def from_url(url): """ Detected filename as unicode or None """ filename = os.path.basename(urlparse.urlparse(url).path) if len(filename.strip(' \n\t.')) == 0: return None return safeunicode(filename) # http://greenbytes.de/tech/tc2231/ @staticmethod def from_headers(headers): """ Detect filename from Content-Disposition headers if present. headers: as dict, list or string """ if not headers: return None if isinstance(headers, string_types): headers = [line.split(':', 1) for line in headers.splitlines()] if isinstance(headers, list): headers = dict(headers) cdisp = headers.get("Content-Disposition") if not cdisp: return None cdtype = cdisp.split(';') if len(cdtype) == 1: return None if cdtype[0].strip().lower() not in ('inline', 'attachment'): return None # several filename params is illegal, but just in case fnames = [x for x in cdtype[1:] if x.strip().startswith('filename=')] if len(fnames) > 1: return None name = fnames[0].split('=')[1].strip(' \t"') name = os.path.basename(name) if not name: return None return name @classmethod def from_any(cls, dst=None, headers=None, url=None): return dst or cls.from_headers(headers) or cls.from_url(url) # http://bitbucket.org/techtonik/python-wget/ def download(url, dst=None): """ High level function, which downloads URL into tmp file in current directory and then renames it to filename autodetected from either URL or HTTP headers. url: which url to download dst: filename or directory of destination """ # detect of dst is a directory dst_ = None if dst and os.path.isdir(dst): dst_ = dst dst = None # get filename for temp file in current directory prefix = FileName.from_any(dst=dst, url=url) fd, tmpfile = tempfile.mkstemp(".tmp", prefix=prefix, dir=".") os.close(fd) os.unlink(tmpfile) if PY2: binurl = url else: # Python 3 can not quote URL as needed binurl = list(urlparse.urlsplit(url)) binurl[2] = urlparse.quote(binurl[2]) binurl = urlparse.urlunsplit(binurl) tmpfile, headers = urlib.urlretrieve(binurl, tmpfile) filename = FileName.from_any(dst=dst, headers=headers, url=url) if dst_: filename = os.path.join(dst_, filename) if os.path.exists(filename): os.unlink(filename) shutil.move(tmpfile, filename) return filename def check_connect(ip, port, retry=1, timeout=0.5): """ Check whether given ``ip`` and ``port`` could connect or not. It will ``retry`` and ``timeout`` on given. """ while retry: try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) except socket.error as e: logger.exception(e) retry -= 1 continue try: s.settimeout(timeout) s.connect((ip, port)) return s.getsockname()[0] except socket.error: logger.error("Connect to ip:%s port:%d fail", ip, port) s.close() finally: retry -= 1 return None def update_query_params(url, params): """ Update query params of given url and return new url. """ parts = list(urlparse.urlparse(url)) query = dict(urlparse.parse_qsl(parts[4])) query.update(params) parts[4] = urlencode(query) new_url = urlparse.urlunparse(parts) return new_url def cookies_str_to_dict(cookies): """ Convert cookies from str to dict. """ if not isinstance(cookies, str): raise TypeError('Invalid type of cookies_string !') cookies_obj = {} for item in cookies.split(';'): item = item.strip().replace('\t', '').replace('\n', '') if '=' not in item: continue key, value = item.split('=', 1) cookies_obj[key] = value return cookies_obj ================================================ FILE: pydu/set.py ================================================ # coding: utf-8 import collections class OrderedSet(object): """ A set which keeps the ordering of the inserted items. """ def __init__(self, iterable=None): self.dict = collections.OrderedDict.fromkeys(iterable or ()) def add(self, item): self.dict[item] = None def remove(self, item): del self.dict[item] def discard(self, item): try: self.remove(item) except KeyError: pass def __iter__(self): return iter(self.dict) def __contains__(self, item): return item in self.dict def __bool__(self): return bool(self.dict) def __nonzero__(self): return bool(self.dict) def __len__(self): return len(self.dict) ================================================ FILE: pydu/slot.py ================================================ from .compat import iteritems, izip class SlotBase(object): """ Base class for class using __slots__. If some args or kwargs are not given when initialize class, the value of them will be set with ``None``. """ def __init__(self, *args, **kwargs): setted = set() kwargs_ = dict(izip(self.__slots__, args)) kwargs_.update(kwargs) for key, value in iteritems(kwargs_): setattr(self, key, value) setted.add(key) for key in set(self.__slots__) - setted: setattr(self, key, None) ================================================ FILE: pydu/string.py ================================================ # coding: utf-8 import locale from .compat import text_type preferredencoding = locale.getpreferredencoding() def safeunicode(obj, encoding='utf-8'): """ Converts any given object to unicode string. >>> safeunicode('hello') u'hello' >>> safeunicode(2) u'2' >>> safeunicode('\xe4\xb8\xad\xe6\x96\x87') u'中文' """ t = type(obj) if t is text_type: return obj elif t is bytes: return obj.decode(encoding) else: return text_type(obj) def safeencode(obj, encoding='utf-8'): """ Converts any given object to encoded string (default: utf-8). >>> safestr('hello') 'hello' >>> safestr(2) '2' """ t = type(obj) if t is text_type: return obj.encode(encoding) elif t is bytes: return obj else: return text_type(obj).encode(encoding) iters = [list, tuple, set, frozenset] class _hack(tuple): pass iters = _hack(iters) iters.__doc__ = """ A list of iterable items (like lists, but not strings). Includes whichever of lists, tuples, sets, and Sets are available in this version of Python. """ def _strips(direction, text, remove): if isinstance(remove, iters): for subr in remove: text = _strips(direction, text, subr) return text if direction == 'l': if text.startswith(remove): return text[len(remove):] elif direction == 'r': if text.endswith(remove): return text[:-len(remove) or None] else: raise ValueError('Direction needs to be r or l.') return text def rstrips(text, remove): """ removes the string `remove` from the right of `text` >>> rstrips('foobar', 'bar') 'foo' """ return _strips('r', text, remove) def lstrips(text, remove): """ removes the string `remove` from the left of `text` >>> lstrips('foobar', 'foo') 'bar' >>> lstrips('FOOBARBAZ', ['FOO', 'BAR']) 'BAZ' >>> lstrips('FOOBARBAZ', ['BAR', 'FOO']) 'BARBAZ' """ return _strips('l', text, remove) def strips(text, remove): """ removes the string `remove` from the both sides of `text` >>> strips('foobarfoo', 'foo') 'bar' """ return rstrips(lstrips(text, remove), remove) def common_prefix(l): """ Return common prefix of the stings >>> common_prefix(['abcd', 'abc1']) 'abc' """ commons = [] for i in range(min(len(s) for s in l)): common = l[0][i] for c in l[1:]: if c[i] != common: return ''.join(commons) commons.append(common) return ''.join(commons) def common_suffix(l): """ Return common suffix of the stings >>> common_suffix(['dabc', '1abc']) 'abc' """ commons = [] for i in range(min(len(s) for s in l)): common = l[0][-i-1] for c in l[1:]: if c[-i-1] != common: return ''.join(reversed(commons)) commons.append(common) return ''.join(reversed(commons)) def sort(s, reverse=False): """ Sort given string by ascending order. If reverse is True, sorting given string by descending order. """ return ''.join(sorted(s, reverse=reverse)) ================================================ FILE: pydu/system.py ================================================ import os import sys import stat import shutil import locale from . import logger from .platform import WINDOWS from .compat import PY2, builtins _openfiles = set() _origin_open = builtins.open if PY2: _origin_file = builtins.file class _trackfile(builtins.file): def __init__(self, *args): self.path = args[0] logger.debug('Opening "%s"', self.path) super(_trackfile, self).__init__(*args) _openfiles.add(self) def close(self): logger.debug('Closing "%s"', self.path) super(_trackfile, self).close() _openfiles.remove(self) def _trackopen(*args): return _trackfile(*args) else: def _trackopen(*args, **kwargs): f = _origin_open(*args, **kwargs) path = args[0] logger.debug('Opening "%s"', path) _openfiles.add(f) origin_close = f.close def close(): logger.debug('Closing "%s"', path) origin_close() _openfiles.remove(f) f.close = close return f class FileTracker(object): @staticmethod def track(): builtins.open = _trackopen if PY2: builtins.file = _trackfile @staticmethod def untrack(): builtins.open = _origin_open if PY2: builtins.file = _origin_file @staticmethod def get_openfiles(): return _openfiles def makedirs(path, mode=0o755, ignore_errors=False, exist_ok=False): """ Create a leaf directory and all intermediate ones. Based on os.makedirs, but also supports ignore_errors which will ignore all errors raised by os.makedirs. """ if exist_ok and os.path.exists(path): return try: os.makedirs(path, mode) except: if not ignore_errors: raise OSError('Create dir: {!r} error.'.format(path)) def remove(path, ignore_errors=False, onerror=None): """ Remove a file or directory. If ignore_errors is set, errors are ignored; otherwise, if onerror is set, it is called to handle the error with arguments (func, path, exc_info) where func is platform and implementation dependent; path is the argument to that function that caused it to fail; and exc_info is a tuple returned by sys.exc_info(). If ignore_errors is False and onerror is None, it attempts to set path as writeable and then proceed with deletion if path is read-only, or raise an exception if path is not read-only. """ if ignore_errors: def onerror(func, path, exc): pass elif onerror is None: def onerror(func, path, exc): try: if (os.stat(path).st_mode & stat.S_IREAD) or not os.access(path, os.W_OK): os.chmod(path, stat.S_IWRITE | stat.S_IWUSR) func(path) else: exc_type, exc_exception, exc_tb = exc raise exc_exception except Exception as e: raise OSError('Remove path: {!r} error. Reason: {}'.format(path, e)) if os.path.isdir(path): shutil.rmtree(path, ignore_errors=ignore_errors, onerror=onerror) else: try: os.remove(path) except: onerror(os.remove, path, sys.exc_info()) def removes(paths, ignore_errors=False, onerror=None): """ Remove a list of file and/or directory. If ignore_errors is set, errors are ignored; otherwise, if onerror is set, it is called to handle the error with arguments (func, path, exc_info) where func is platform and implementation dependent; path is the argument to that function that caused it to fail; and exc_info is a tuple returned by sys.exc_info(). If ignore_errors is False and onerror is None, an exception is raised. """ for path in paths: remove(path, ignore_errors=ignore_errors, onerror=onerror) def open_file(path, mode='wb+', buffer_size=-1, ignore_errors=False): """ Open a file, defualt mode 'wb+'. If path not exists, it will be created automatically. If ignore_errors is set, errors are ignored. """ f = None try: if path and not os.path.isdir(path): makedirs(os.path.dirname(path), exist_ok=True) f = open(path, mode, buffer_size) except: if not ignore_errors: raise OSError('Open file: {!r} error'.format(path)) return f def copy(src, dst, ignore_errors=False, follow_symlinks=True): """ Copy data and mode bits ("cp src dst"). Both the source and destination may be a directory. When copy a directory,which contains a symlink, If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic links are copied. If the file pointed by the symlink doesn't exist, an exception will be raise. When copy a file,if follow_symlinks is false and src is a symbolic link, a new symlink will be created instead of copying the file it points to,else the contents of the file pointed to by symbolic links is copied. If source and destination are the same file, a SameFileError will be raised. If ignore_errors is set, errors are ignored. """ try: if os.path.isdir(src): shutil.copytree(src, dst, symlinks=follow_symlinks) else: if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: shutil.copy(src, dst) except: if not ignore_errors: raise OSError('Copy {!r} to {!r} error'.format(src, dst)) def touch(path): """ Open a file as write,and then close it. """ with open(path, 'w'): pass def chmod(path, mode, recursive=False): """ Change permissions to the given mode. If `recursive` is True perform recursively. >>> chmod('/opt/sometest', 0o755) >>> oct(os.stat('/opt/sometest').st_mode)[-3:] 755 """ chmod_ = os.chmod if recursive and os.path.isdir(path): for dirpath, _, filenames in os.walk(path): chmod_(dirpath, mode) for filename in filenames: chmod_(os.path.join(dirpath, filename), mode) else: os.chmod(path, mode) if PY2: # shutil.which from Python3 def which(cmd, mode=os.F_OK | os.X_OK, path=None): """ Given a command, mode, and a PATH string, return the path which conforms to the given mode on the PATH, or None if there is no such file. `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result of os.environ.get("PATH"), or can be overridden with a custom search path. """ # Check that a given file can be accessed with the correct mode. # Additionally check that `file` is not a directory, as on Windows # directories pass the os.access check. def _access_check(fn, mode): return (os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn)) # If we're given a path with a directory part, look it up directly rather # than referring to PATH directories. This includes checking relative to the # current directory, e.g. ./script if os.path.dirname(cmd): if _access_check(cmd, mode): return cmd return None if path is None: path = os.environ.get("PATH", os.defpath) if not path: return None path = path.split(os.pathsep) if WINDOWS: # The current directory takes precedence on Windows. if not os.curdir in path: path.insert(0, os.curdir) # PATHEXT is necessary to check on Windows. pathext = os.environ.get("PATHEXT", "").split(os.pathsep) # See if the given file matches any of the expected path extensions. # This will allow us to short circuit when given "python.exe". # If it does match, only test that one, otherwise we have to try # others. if any(cmd.lower().endswith(ext.lower()) for ext in pathext): files = [cmd] else: files = [cmd + ext for ext in pathext] else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. files = [cmd] seen = set() for dir in path: normdir = os.path.normcase(dir) if not normdir in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) if _access_check(name, mode): return name return None else: which = shutil.which if WINDOWS: # For Windows system from ctypes import windll class chcp(object): """ Context manager which sets the active code page number. It could also be used as function. """ def __init__(self, code): self.origin_code = windll.kernel32.GetConsoleOutputCP() self.code = code windll.kernel32.SetConsoleOutputCP(code) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): windll.kernel32.SetConsoleOutputCP(self.origin_code) def __repr__(self): return ''.format(self.code) else: # For non Windows system def symlink(src, dst, overwrite=False, ignore_errors=False): """ Create a symbolic link pointing to source named link_name. If dist is exist and overwrite is true,a new symlink will be created If ignore_errors is set, errors are ignored. """ try: if os.path.exists(dst): if overwrite: remove(dst) else: return os.symlink(src, dst) except Exception: if not ignore_errors: raise OSError('Link {!r} to {!r} error'.format(dst, src)) def link(src, dst, overwrite=False, ignore_errors=False): """ Create a hard link pointing to source named link_name. If dist is exist and overwrite is true,a new symlink will be created If ignore_errors is set, errors are ignored. """ try: if os.path.exists(dst): if overwrite: remove(dst) else: return os.link(src, dst) except: if not ignore_errors: raise OSError('Link {!r} to {!r} error'.format(dst, src)) def preferredencoding(): """ Get best encoding for the system. """ try: encoding = locale.getpreferredencoding() 'test encoding'.encode(encoding) except UnicodeEncodeError: encoding = 'UTF-8' return encoding ================================================ FILE: pydu/unit.py ================================================ BYTE_UNITS = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB') class Bytes(object): """ Supply several methods dealing with bytes. """ def __init__(self, bytes): self.bytes = bytes def convert(self, unit=None, multiple=1024): """ Convert bytes with given ``unit``. If `unit` is None, convert bytes with suitable unit. Convert `multiple` is default to be 1024. """ step = 0 if not unit: while self.bytes >= multiple and step < len(BYTE_UNITS) - 1: self.bytes /= multiple step += 1 unit = BYTE_UNITS[step] else: # convert to specific unit index_of_unit = BYTE_UNITS.index(unit) while len(BYTE_UNITS) - 1 > step and index_of_unit != step: self.bytes /= multiple step += 1 return self.bytes, unit ================================================ FILE: requirements-dev.txt ================================================ pytest>=2.8.0 pytest-xdist coverage psutil ================================================ FILE: setup.cfg ================================================ [bdist_wheel] universal = 1 [metadata] license_file = LICENSE.txt ================================================ FILE: setup.py ================================================ import sys from pydu import __version__ from pydu.compat import PY2 from setuptools import setup, find_packages from setuptools.command.test import test as TestCommand class PyTest(TestCommand): user_options = [('pytest-args=', 'a', "Arguments to pass into py.test")] def initialize_options(self): TestCommand.initialize_options(self) try: from multiprocessing import cpu_count self.pytest_args = ['-n', str(cpu_count())] except (ImportError, NotImplementedError): self.pytest_args = ['-n', '1'] def finalize_options(self): TestCommand.finalize_options(self) self.test_args = [] self.test_suite = True def run_tests(self): import pytest errno = pytest.main(self.pytest_args) sys.exit(errno) test_requirements = [] for line in open('requirements-dev.txt'): requirement = line.strip() if requirement: test_requirements.append(requirement) open_kwargs = {} if PY2 else {'encoding': 'utf-8'} setup( name="pydu", version=__version__, description="Useful data structures, utils for Python.", long_description=open('README.md', **open_kwargs).read(), long_description_content_type='text/markdown', author="Prodesire", author_email='wangbinxin001@126.com', license='MIT License', url="https://github.com/Prodesire/pydu", cmdclass={'test': PyTest}, tests_require=test_requirements, packages=find_packages(), classifiers=[ 'Operating System :: OS Independent', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', 'Programming Language :: Python :: Implementation', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Topic :: Software Development :: Libraries' ], ) ================================================ FILE: stubs/pydu/__init__.pyi ================================================ ================================================ FILE: stubs/pydu/archive.pyi ================================================ from tarfile import TarFile from zipfile import ZipFile from typing import List class Archive(object): _archive = ... # type: BaseArchive def __init__(self, file: file, ext: str='') -> None: ... def _archive_cls(self, file: file, ext: str='') -> None: ... def extract(self, dst: str='') -> None: ... def list(self) -> None: ... def filenames(self) -> list: ... def close(self) -> None: ... class BaseArchive(object): @staticmethod def _copy_permissions(mode: int, filename: str) -> None: ... def split_leading_dir(self, path: str) -> None: ... def has_leading_dir(self, paths: List[str]) -> None: ... def extract(self, dst: str) -> None: ... def list(self) -> list: ... def filenames(self) -> List[str]: ... class TarArchive(BaseArchive): _archive = ... # type: TarFile def extract(self, dst: str) -> None: ... def list(self) -> None: ... def filenames(self) -> List[str]: ... def close(self) -> None: ... class ZipArchive(BaseArchive): _archive = ... # type: ZipFile def extract(self, dst: str) -> None: ... def list(self) -> None: ... def filenames(self) -> List[str]: ... def close(self) -> None: ... ================================================ FILE: stubs/pydu/cmd.pyi ================================================ """Stubs for cmd""" from typing import Tuple, List, Union class TimeoutExpired(Exception): def __init__(self, cmd: str, timeout: Union[int, float], output: dict=None, stderr: dict=None) -> None: ... def run(cmd: str, shell: bool=..., env: dict=None, timeout: Union[int, float]=..., timeinterval: Union[int, float]=...) -> Tuple[int, str]: ... def run_with_en_env(cmd: str, shell: bool=..., env: dict=None, timeout: Union[int, float]=..., timeinterval: Union[int, float]=...) -> Tuple[int, str]: ... def terminate(pid: int) -> None: ... def cmdline_argv() -> List: ... ================================================ FILE: stubs/pydu/console.pyi ================================================ """Stubs for console""" from typing import Tuple def console_size(fallback: Tuple[int, int]=...) -> Tuple[int, int]: ... ================================================ FILE: stubs/pydu/convert.pyi ================================================ from typing import Any def boolean(obj: Any) -> bool: ... def bin2oct(x: str) -> str: ... def bin2dec(x: str) -> int: ... def bin2hex(x: str) -> str: ... def oct2bin(x: str) -> str: ... def oct2dec(x: str) -> int: ... def oct2hex(x: str) -> str: ... def dec2bin(x: int) -> str: ... def dec2oct(x: int) -> str: ... def dec2hex(x: int) -> str: ... def hex2bin(x: str) -> str: ... def hex2oct(x: str) -> str: ... def hex2dec(x: str) -> int: ... ================================================ FILE: stubs/pydu/dict.pyi ================================================ import collections from typing import Iterable, Tuple, Any class CaseInsensitiveDict(collections.MutableMapping): _store = ... # type: dict def __init__(self, data: dict=None, **kwargs) -> None: ... def lower_items(self) -> Iterable[Tuple[str, Any]]: ... ================================================ FILE: stubs/pydu/dt.pyi ================================================ from typing import Callable class timer(object): elapsed = ... # type: float print_func = ... # type: Callable ================================================ FILE: stubs/pydu/environ.pyi ================================================ from typing import Dict, ContextManager, Union StrList = Union[str, list] def environ(kwargs: Dict[str, str]) -> ContextManager[None]: ... def path(append: StrList, prepend: StrList, replace: StrList) -> ContextManager[None]: ... ================================================ FILE: stubs/pydu/exception.pyi ================================================ from typing import ContextManager, Type, List, Any, Callable from pydu.compat import PY2 if PY2: def ignore(*exceptions: Type[BaseException]) -> ContextManager[None]: ... else: class ignore(ContextManager[None]): def __init__(self, *exceptions: Type[BaseException]) -> None: ... def default_if_except(exceptions_clses: List[Exception], default: Any=None) -> Callable: ... ================================================ FILE: stubs/pydu/functional.pyi ================================================ from typing import List, Callable, Any def compose(*funcs: List[Callable]): return Any ================================================ FILE: stubs/pydu/iter.pyi ================================================ from typing import Iterable, TypeVar, Optional, Callable T = TypeVar('T') def first(iterable: Iterable[T]) -> T: ... def last(iterable: Iterable[T]) -> Optional[T]: ... def all(iterable: Iterable[T], predicate: Callable[[T], bool]) -> bool: ... def any(iterable: Iterable[T], predicate: Callable[[T], bool]) -> bool: ... def join(iterable: Iterable[T], separator: str) -> str: ... ================================================ FILE: stubs/pydu/list.pyi ================================================ from typing import Callable, Any, Hashable, Iterable KeyFunc = Callable[[Any], Hashable] def uniq(seq: Iterable[Any], key: KeyFunc=None) -> list: ... def tolist(obj: Any) -> list: ... def flatten(seq: Iterable[Any]) -> Iterable[Any]: ... ================================================ FILE: stubs/pydu/misc.pyi ================================================ from typing import Callable, Any AnyCallable = Callable[..., Any] def timeout(seconds: int, error_message: str) -> Callable[[AnyCallable], AnyCallable]: ... def trace(func: AnyCallable) -> AnyCallable: ... def memoize(func: AnyCallable) -> AnyCallable: ... def memoize_when_activated(func: AnyCallable) -> AnyCallable: ... def super_len(obj: Any) -> int: ... ================================================ FILE: stubs/pydu/network.pyi ================================================ from typing import Union def dotted_netmask(mask: Union[int, str]) -> str: ... def is_ipv4(ip: str) -> bool: ... def is_ipv6(ip: str) -> bool: ... def ip2int(ip_str) -> int: ... def int2ip(ip_int: int) -> int: ... ================================================ FILE: stubs/pydu/path.pyi ================================================ from typing import ContextManager, List def cd(path: str) -> ContextManager[None]: ... def is_super_path(path1: str, path2: str) -> bool: ... def normjoin(path: str, *paths: List(str)) -> str: ... def filename(path: str) -> str: ... def fileexe(path: str) -> str: ... ================================================ FILE: stubs/pydu/process.pyi ================================================ from typing import List, Dict, Union def get_processes_by_path(path: str) -> List[Dict[str, Union[int, str]]]: ... ================================================ FILE: stubs/pydu/request.pyi ================================================ from typing import Union, Optional Headers = Union[dict, list, str] class FileName(object): def from_url(self, url: str) -> str: None def from_headers(self, headers: Headers) -> Optional[str]: ... def from_any(cls, dst: str=None, headers: Headers=None, url: str=None) -> Optional[str]: ... def download(url: str, dst: str=None) -> str: ... def check_connect(ip: str, port: int, retry: int=1, timout: float=0.5) -> Optional[str]: ... def update_query_params(url: str, params: dict) -> str: ... def cookies_str_to_dict(cookies: str) -> dict: ... ================================================ FILE: stubs/pydu/set.pyi ================================================ from typing import Iterable, Tuple, Any class OrderedSet(object): def __init__(self, iterable: Iterable[Tuple[Any, Any]]=None) -> None: ... ================================================ FILE: stubs/pydu/string.pyi ================================================ from typing import Any from pydu.compat import text_type def safeunicode(obj: Any, encoding: str) -> text_type: ... def safeencode(obj: Any, encoding: str) -> bytes: ... def _strips(direction: str, text: str, remove: str) -> str: ... def rstrips(text: str, remove: str) -> str: ... def lstrips(text: str, remove: str) -> str: ... def strips(text: str, remove: str) -> str: ... def common_prefix(l: list) -> str: ... def common_suffix(l: list) -> str: ... def sort(s: str, reverse: bool) -> str: ... ================================================ FILE: stubs/pydu/system.pyi ================================================ import os from typing import Callable, List from pydu.platform import WINDOWS def makedirs(path: str, mode: int=0o755, ignore_errors: bool=False, exist_ok: bool=False) -> None: ... def remove(path: str, ignore_errors: bool=False, onerror: Callable=None) -> None: ... def removes(paths: List[str], ignore_errors: bool=False, onerror: Callable=None) -> None: ... def open_file(path: str, mode: str='wb+', buffer_size: int=-1, ignore_errors: bool=False) -> None: ... def copy(src: str, dst: str, ignore_errors: bool=False, follow_symlinks: bool=True) -> None: ... def touch(path: str) -> None: ... def chmod(path: str, mode: int, recursive: bool=False) -> None: ... def which(cmd: str, mode: int=os.F_OK | os.X_OK, path: str=None) -> None: ... if WINDOWS: class chcp(object): def __init__(self, code: str) -> None: ... else: def symlink(src: str, dst: str, overwrite: bool=False, ignore_errors: bool=False) -> None: ... def link(src: str, dst: str, overwrite: bool=False, ignore_errors: bool=False) -> None: ... ================================================ FILE: stubs/pydu/unit.pyi ================================================ from typing import Tuple class Bytes(object): bytes=... # type: str def __init__(self, bytes: str) -> None: ... def convert(self, unit: str=None, multiple: int=1024) -> Tuple[str, str]: ... ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/files/bad/unrecognized.txt ================================================ File with unrecognized archive extension. ================================================ FILE: tests/test_archive.py ================================================ # coding: utf-8 import os import shutil import tempfile import unittest from os.path import isfile, join as pathjoin from pydu.archive import extract, UnrecognizedArchiveFormat TEST_DIR = os.path.dirname(os.path.realpath(__file__)) class TempDirMixin(object): """ Mixin class for TestCase subclasses to set up and tear down a temporary directory for unpacking archives during tests. """ def setUp(self): """ Create temporary directory for testing extraction. """ self.tmpdir = tempfile.mkdtemp() os.chdir(TEST_DIR) def tearDown(self): """ Clean up temporary directory. """ shutil.rmtree(self.tmpdir) def check_files(self, tmpdir): self.assertTrue(isfile(pathjoin(tmpdir, '1'))) self.assertTrue(isfile(pathjoin(tmpdir, '2'))) self.assertTrue(isfile(pathjoin(tmpdir, 'foo', '1'))) self.assertTrue(isfile(pathjoin(tmpdir, 'foo', '2'))) self.assertTrue(isfile(pathjoin(tmpdir, 'foo', 'bar', '1'))) self.assertTrue(isfile(pathjoin(tmpdir, 'foo', 'bar', '2'))) class ArchiveTester(TempDirMixin): """ A mixin class to be used for testing many Archive methods for a single archive file. """ archive = None ext = '' def setUp(self): super(ArchiveTester, self).setUp() self.archive_path = pathjoin(TEST_DIR, 'files', self.archive) def test_extract(self): extract(self.archive_path, self.tmpdir, ext=self.ext) self.check_files(self.tmpdir) def test_extract_fileobject(self): with open(self.archive_path, 'rb') as f: extract(f, self.tmpdir, ext=self.ext) self.check_files(self.tmpdir) def test_extract_no_to_path(self): cur_dir = os.getcwd() os.chdir(self.tmpdir) extract(self.archive_path, ext=self.ext) self.check_files(self.tmpdir) os.chdir(cur_dir) def test_extract_bad_fileobject(self): class File: pass f = File() self.assertRaises(UnrecognizedArchiveFormat, extract, (f, self.tmpdir), {'ext': self.ext}) class TestZip(ArchiveTester, unittest.TestCase): archive = 'foobar.zip' class TestTar(ArchiveTester, unittest.TestCase): archive = 'foobar.tar' class TestGzipTar(ArchiveTester, unittest.TestCase): archive = 'foobar.tar.gz' class TestBzip2Tar(ArchiveTester, unittest.TestCase): archive = 'foobar.tar.bz2' class TestNonAsciiNamedTar(ArchiveTester, unittest.TestCase): archive = u'压缩.tgz' class TestUnicodeNamedZip(ArchiveTester, unittest.TestCase): archive = u'压缩.zip' class TestExplicitExt(ArchiveTester, unittest.TestCase): archive = 'foobar_tar_gz' ext = '.tar.gz' ================================================ FILE: tests/test_cmd.py ================================================ import sys import pytest import time import subprocess from pydu.compat import string_types from pydu.string import safeunicode from pydu.cmd import TimeoutExpired, run, run_with_en_env, terminate, cmdline_argv def test_run(): retcode, output = run('echo hello', shell=True) assert retcode == 0 assert safeunicode(output).rstrip('\r\n') == 'hello' with pytest.raises(TimeoutExpired) as e: cmd = '{} -c "import time; time.sleep(1)"'.format(sys.executable) timeout = 0.2 run(cmd, shell=True, timeout=timeout, timeinterval=0.05) assert e.cmd == cmd assert e.timeout == timeout assert hasattr(e, 'output') assert hasattr(e, 'stderr') def test_run_with_en_env(): _, output = run_with_en_env('nocmd', shell=True) assert output.decode('ascii') _, output = run_with_en_env(['nocmd'], shell=True) assert output.decode('ascii') def test_terminate(): p = subprocess.Popen('{} -c "import time; time.sleep(1)"'.format(sys.executable), shell=True) terminate(p.pid) time.sleep(0.1) assert p.poll() is not None def test_cmdline_argv(): argv = cmdline_argv() for s in argv[1:]: assert isinstance(s, string_types) ================================================ FILE: tests/test_compat.py ================================================ from pydu.compat import (PY2, iterkeys, itervalues, iteritems, text_type, string_types, numeric_types, is_iterable, has_next_attr, imap, cmp) def test_itersth(): d = dict(a=1, b=2) for key in iterkeys(d): assert key in ('a', 'b') for value in itervalues(d): assert value in (1, 2) for items in iteritems(d): assert items in (('a', 1), ('b', 2)) def test_has_next_attr(): if PY2: class NextAttr: def next(self): pass else: class NextAttr: def __next__(self): pass assert has_next_attr(NextAttr()) assert not has_next_attr('') def test_is_iterable(): assert is_iterable(list()) assert is_iterable(tuple()) assert is_iterable(dict()) assert is_iterable(set()) assert not is_iterable(1) def test_types(): assert isinstance(u'a', text_type) assert isinstance(u'a', string_types) assert isinstance('a', string_types) assert isinstance(1, numeric_types) assert isinstance(2**50, numeric_types) def test_urlmisc(): from pydu.compat import urljoin, urlib, urlparse def test_imap(): assert list(imap(pow, (2, 3, 10), (5, 2, 3))) == [32, 9, 1000] assert list(imap(max, (1, 4, 7), (2, 3, 8))) == [2, 4, 8] def test_cmp(): assert cmp(1, 2) < 0 assert cmp(1, 1) == 0 assert cmp(2, 1) > 0 ================================================ FILE: tests/test_console.py ================================================ from pydu.console import console_size def test_console_size(): size = console_size() assert isinstance(size, tuple) assert len(size) == 2 ================================================ FILE: tests/test_convert.py ================================================ import pytest from pydu.convert import (boolean, bin2oct, bin2dec, bin2hex, oct2bin, oct2dec, oct2hex, dec2bin, dec2oct, dec2hex, hex2bin, hex2oct, hex2dec) BIG_NUM_STR = '10'*50 BIG_NUM = 10**50 class TestBoolean: def test_accepted_text(self): for text in ('yes', 'y', 'on', 'true', 't', '1'): assert boolean(text) assert boolean(text.upper()) for text in ('no', 'n', 'off', 'false', 'f', '0'): assert not boolean(text) assert not boolean(text.upper()) @pytest.mark.parametrize('text', ('a', 'b')) def test_unaccepted_text(self, text): with pytest.raises(ValueError): boolean(text) def test_nonstring(self): for obj in (10, [1], {1: 1}): assert boolean(obj) for obj in (0, [], {}): assert not boolean(obj) def test_bin2oct(): assert bin2oct('1001') == '11' assert 'L' not in bin2oct(BIG_NUM_STR) def test_bin2dec(): assert bin2dec('11') == 3 def test_bin2hex(): assert bin2hex('11010') == '1a' assert 'L' not in bin2hex(BIG_NUM_STR) def test_oct2bin(): assert oct2bin('11') == '1001' assert 'L' not in oct2bin(BIG_NUM_STR) def test_oct2dec(): assert oct2dec('11') == 9 def test_oct2hex(): assert oct2hex('32') == '1a' assert 'L' not in oct2hex(BIG_NUM_STR) def test_dec2bin(): assert dec2bin(3) == '11' assert 'L' not in dec2bin(BIG_NUM) def test_dec2oct(): assert dec2oct(9) == '11' assert 'L' not in dec2oct(BIG_NUM) def test_dec2hex(): assert dec2hex(26) == '1a' assert 'L' not in dec2hex(BIG_NUM) def test_hex2bin(): assert hex2bin('1a') == '11010' assert 'L' not in hex2bin(BIG_NUM_STR) def test_hex2oct(): assert hex2oct('1a') == '32' assert 'L' not in hex2oct(BIG_NUM_STR) def test_hex2dec(): assert hex2dec('1a') == 26 ================================================ FILE: tests/test_dict.py ================================================ import pytest import unittest from pydu.dict import AttrDict, LookupDict, CaseInsensitiveDict, OrderedDefaultDict, attrify class TestAttrDict: def test_attr_access_with_init(self): d = AttrDict(key=1) assert d['key'] == 1 assert d.key == 1 def test_attr_access_without_init(self): d = AttrDict() d['key'] = 1 assert d['key'] == 1 assert d.key == 1 d.anotherkey = 1 assert d.anotherkey == 1 assert d['anotherkey'] == 1 def test_attr_delete(self): d = AttrDict(key=1) del d.key with pytest.raises(AttributeError): del d.key def test_repr(self): d = AttrDict() assert repr(d) == '' class TestLooUpDict: def test_key_exist(self): d = LookupDict() d['key'] = 1 assert d['key'] == 1 def test_key_not_exist(self): d = LookupDict() assert d['key'] is None class TestCaseInsensitiveDict(unittest.TestCase): def setUp(self): self.d = CaseInsensitiveDict() self.d['Accept'] = 1 def test_ci_dict_set(self): assert self.d['aCCept'] == 1 assert list(self.d) == ['Accept'] def test_ci_dict_del(self): del self.d['accept'] assert not self.d def test_ci_dict_copy_and_equal(self): d = self.d.copy() assert d == self.d class TestOrderedDefaultDict: def test_default_normal(self): d = OrderedDefaultDict(int) assert d[1] == 0 assert d['a'] == 0 d[2] = 2 assert d[2] == 2 assert list(d.keys()) == [1, 'a', 2] d = OrderedDefaultDict(int, a=1) assert d['a'] == 1 def test_default_factory_not_callable(self): with pytest.raises(TypeError): OrderedDefaultDict('notcallable') def test_default_factory_none(self): d = OrderedDefaultDict() with pytest.raises(KeyError): d[1] def test_copy(self): d1 = OrderedDefaultDict(int, a=[]) d2 = d1.copy() assert d2['a'] == [] d1['a'].append(1) assert d2['a'] == [1] def test_deepcopy(self): import copy d1 = OrderedDefaultDict(int, a=[]) d2 = copy.deepcopy(d1) assert d2['a'] == [] d1['a'].append(1) assert d2['a'] == [] def test_repr(self): d = OrderedDefaultDict(int, a=1) assert repr(d).startswith('OrderedDefaultDict') def test_attrify(): attrd = attrify({ 'a': [1, 2, {'b': 'b'}], 'c': 'c', }) assert attrd.a == [1, 2, {'b': 'b'}] assert attrd.a[2].b == 'b' assert attrd.c == 'c' attrd = attrify((1, 2)) assert attrd == (1, 2) attrd = attrify({ 'a': 1, 'b': (1, 2) }) assert attrd.a == 1 assert attrd.b == (1, 2) ================================================ FILE: tests/test_dt.py ================================================ import os from pydu.dt import timer class TestTimer(object): def test_context_manager(self): timeit = timer() with timeit: os.getcwd() assert timeit.elapsed is not None def test_decorator(self): timeit = timer() @timeit def foo(): os.getcwd() foo() assert timeit.elapsed is not None def test_print_func(self): import sys timeit = timer(print_func=sys.stdout.write) with timeit: os.getcwd() assert timeit.elapsed is not None ================================================ FILE: tests/test_environ.py ================================================ import os from pydu.environ import environ, path def test_environ(): os.environ['c'] = 'c' with environ(a='a', b='', c=None, d=None): assert os.environ['a'] == 'a' assert os.environ['b'] == '' assert 'c' not in os.environ assert 'd' not in os.environ assert 'a' not in os.environ assert 'b' not in os.environ assert 'c' in os.environ assert 'd' not in os.environ def test_path(): with path(append='foo', prepend='boo'): assert os.environ['PATH'].endswith(os.pathsep + 'foo') assert os.environ['PATH'].startswith('boo' + os.pathsep) assert not os.environ['PATH'].endswith(os.pathsep + 'foo') assert not os.environ['PATH'].startswith('boo' + os.pathsep) with path(append='foo', prepend='boo', replace='replace'): assert os.environ['PATH'] == 'replace' assert os.environ['PATH'] != 'replace' ================================================ FILE: tests/test_exception.py ================================================ from pydu.exception import ignore, default_if_except def test_ignore(): with ignore(ValueError, AttributeError): int('abc') int.no_exists_func() def test_default_if_except(): @default_if_except(ValueError, default=0) def foo(value): return int(value) assert foo('abc') == 0 assert foo('1') == 1 ================================================ FILE: tests/test_functional.py ================================================ from pydu.functional import compose def test_compose(): def f1(a, b=1): return a+b def f2(a): return 2*a def f3(a, b=3): return a+b assert compose(f1, f2, f3)(1) == 9 assert compose(f1, f2, f3)(1, b=5) == 13 ================================================ FILE: tests/test_inspect.py ================================================ from pydu.inspect import (get_func_args, get_func_full_args, func_accepts_var_args, func_accepts_kwargs, func_supports_parameter) class Person: def no_arguments(self): return None def one_argument(self, something): return something def just_args(self, *args): return args def just_kwargs(self, **kwargs): return kwargs def all_kinds(self, name, address='home', age=25, *args, **kwargs): return kwargs def func_no_arguments(): pass def func_one_argument(something): pass def func_just_args(*args): pass def func_just_kwargs(**kwargs): pass def func_all_kinds(name, address='home', age=25, *args, **kwargs): pass def test_get_func_args(): arguments = ['name', 'address', 'age'] assert get_func_args(Person.all_kinds) == arguments def test_get_func_full_args(): # no arguments assert get_func_full_args(Person.no_arguments) == [] assert get_func_full_args(func_no_arguments) == [] # one argument assert get_func_full_args(Person.one_argument) == [('something',)] assert get_func_full_args(func_one_argument) == [('something',)] # all_arguments arguments = [('name',), ('address', 'home'), ('age', 25), ('*args',), ('**kwargs',)] assert get_func_full_args(Person.all_kinds) == arguments assert get_func_full_args(func_all_kinds) == arguments def test_func_accepts_var_args(): # has args assert func_accepts_var_args(Person.just_args) assert func_accepts_var_args(func_just_args) # no args assert not func_accepts_var_args(Person.one_argument) assert not func_accepts_var_args(func_one_argument) def test_func_accepts_kwargs(): # has kwargs assert func_accepts_kwargs(Person.just_kwargs) assert func_accepts_kwargs(func_just_kwargs) # no kwargs assert not func_accepts_kwargs(Person.one_argument) assert not func_accepts_kwargs(func_one_argument) def test_func_supports_parameter(): for all_kinds in Person.all_kinds, func_all_kinds: assert func_supports_parameter(all_kinds, 'name') assert func_supports_parameter(all_kinds, 'kwargs') assert not func_supports_parameter(all_kinds, 'self') ================================================ FILE: tests/test_iter.py ================================================ import pytest from pydu.iter import first, last, all, any, join @pytest.mark.parametrize( 'iterable', ( [1, 2], (1, 2), {1, 2}, {1: 1, 2: 2}, iter([1, 2]) )) def test_first_last(iterable): assert first(iterable) == 1 assert last(iterable) == 2 def test_all(): assert all([0, 1, 2], lambda x: x+1) assert not all([0, 1, 2], lambda x: x) def test_any(): assert any([-1, -1, 0], lambda x: x+1) assert not any([-1, -1, -1], lambda x: x + 1) def test_join(): assert join(iter([1, '2', 3])) == '123' assert join(iter([1, '2', 3]), separator=',') == '1,2,3' ================================================ FILE: tests/test_list.py ================================================ import pytest from pydu.list import uniq, tolist, flatten def test_uniq(): assert uniq([1, 4, 0, 2, 0, 3]) == [1, 4, 0, 2, 3] @pytest.mark.parametrize('obj', ('foo', ['foo'])) def test_tolist(obj): assert tolist(obj) == ['foo'] def test_flatten(): assert list(flatten([1, 2])) == [1, 2] assert list(flatten([1, [2, 3]])) == [1, 2, 3] assert list(flatten([1, [2, [3, 4]]])) == [1, 2, 3, 4] ================================================ FILE: tests/test_misc.py ================================================ import sys import time import pytest from pydu.misc import (trace, TimeoutError, timeout, memoize, memoize_when_activated, super_len) try: from cStringIO import StringIO # py2 except ImportError: from io import StringIO # py3 def test_timeout(): @timeout(1) def f1(): time.sleep(0.01) return 1 @timeout(0.01) def f2(): time.sleep(1) return 2 assert f1() == 1 with pytest.raises(TimeoutError): f2() def test_memoize(): @memoize def foo(*args, **kwargs): """foo docstring""" calls.append(None) return (args, kwargs) calls = [] # no args for x in range(2): ret = foo() expected = ((), {}) assert ret == expected assert len(calls) == 1 # with args for x in range(2): ret = foo(1) expected = ((1,), {}) assert ret == expected assert len(calls) == 2 # with args + kwargs for x in range(2): ret = foo(1, bar=2) expected = ((1,), {'bar': 2}) assert ret == expected assert len(calls) == 3 # clear cache foo.cache_clear() ret = foo() expected = ((), {}) assert ret == expected assert len(calls) == 4 # docstring assert foo.__doc__ == "foo docstring" def test_memoize_when_activated(): class Foo: @memoize_when_activated def foo(self): calls.append(None) f = Foo() calls = [] f.foo() f.foo() assert len(calls) == 2 # activate calls = [] f.foo.cache_activate() f.foo() f.foo() assert len(calls) == 1 # deactivate calls = [] f.foo.cache_deactivate() f.foo() f.foo() assert len(calls) == 2 class TestSuperLen: @pytest.mark.parametrize( 'stream, value', ( (StringIO, 'Test'), )) def test_io_streams(self, stream, value): """Ensures that we properly deal with different kinds of IO streams.""" assert super_len(stream()) == 0 assert super_len(stream(value)) == 4 def test_super_len_correctly_calculates_len_of_partially_read_file(self): """Ensure that we handle partially consumed file like objects.""" s = StringIO() s.write('foobarbogus') assert super_len(s) == 0 @pytest.mark.parametrize('error', [IOError, OSError]) def test_super_len_handles_files_raising_weird_errors_in_tell(self, error): """If tell() raises errors, assume the cursor is at position zero.""" class BoomFile(object): def __len__(self): return 5 def tell(self): raise error() assert super_len(BoomFile()) == 0 @pytest.mark.parametrize('error', [IOError, OSError]) def test_super_len_tell_ioerror(self, error): """Ensure that if tell gives an IOError super_len doesn't fail""" class NoLenBoomFile(object): def tell(self): raise error() def seek(self, offset, whence): pass assert super_len(NoLenBoomFile()) == 0 def test_string(self): assert super_len('Test') == 4 @pytest.mark.parametrize( 'mode, warnings_num', ( ('r', 0), ('rb', 0), )) def test_file(self, tmpdir, mode, warnings_num, recwarn): file_obj = tmpdir.join('test.txt') file_obj.write('Test') with file_obj.open(mode) as fd: assert super_len(fd) == 4 assert len(recwarn) == warnings_num def test_super_len_with__len__(self): foo = [1, 2, 3, 4] len_foo = super_len(foo) assert len_foo == 4 def test_super_len_with_no__len__(self): class LenFile(object): def __init__(self): self.len = 5 assert super_len(LenFile()) == 5 def test_super_len_with_tell(self): foo = StringIO('12345') assert super_len(foo) == 5 foo.read(2) assert super_len(foo) == 3 def test_super_len_with_fileno(self): with open(__file__, 'rb') as f: length = super_len(f) file_data = f.read() assert length == len(file_data) def test_super_len_with_no_matches(self): """Ensure that objects without any length methods default to 0""" assert super_len(object()) == 0 ================================================ FILE: tests/test_network.py ================================================ import pytest from pydu.network import (dotted_netmask, is_ipv4, is_ipv6, get_free_port, ip2int, int2ip) @pytest.mark.parametrize( 'mask, expected', ( (8, '255.0.0.0'), (24, '255.255.255.0'), (25, '255.255.255.128'), )) def test_dotted_netmask(mask, expected): assert dotted_netmask(mask) == expected class TestIsIPv4Address: def test_valid(self): assert is_ipv4('8.8.8.8') @pytest.mark.parametrize('value', ('8.8.8.8.8', 'localhost.localdomain')) def test_invalid(self, value): assert not is_ipv4(value) class TestIsIPv6Address: def test_valid(self): assert is_ipv6('fe80::9e5b:b149:e187:1a18') @pytest.mark.parametrize('value', ('fe80::9e5b:b149:e187::', 'localhost.localdomain')) def test_invalid(self, value): assert not is_ipv6(value) def test_get_free_port(): port = get_free_port() assert isinstance(port, int) assert 65536 > port > 0 def test_ip2int(): assert ip2int('10.1.1.1') == 167837953 with pytest.raises(ValueError): ip2int('255.255.255.256') assert ip2int('fe80::9e5b:b149:e187:1a18') == 338288524927261089665429805853095434776 with pytest.raises(ValueError): ip2int('fe80::9e5b:b149:e187::') def test_int2ip(): assert int2ip(167837953) == '10.1.1.1' assert int2ip(338288524927261089665429805853095434776) == 'fe80::9e5b:b149:e187:1a18' with pytest.raises(ValueError): int2ip(10**50) ================================================ FILE: tests/test_path.py ================================================ import os import pytest from pydu.platform import WINDOWS from pydu.path import cd, is_super_path, normjoin, filename, fileext def test_cd(tmpdir): path = str(tmpdir) cwd = os.getcwd() with cd(path): assert os.getcwd() == path assert os.getcwd() == cwd class TestIsSupoerPath: def test_is_super_path_general(self): assert is_super_path('/aa/bb/cc', '/aa/bb/cc') assert is_super_path('/aa/bb', '/aa/bb/cc') assert is_super_path('/aa', '/aa/bb/cc') assert is_super_path('/', '/aa/bb/cc') assert is_super_path('/', '/') assert not is_super_path('/a', '/aa/bb/cc') @pytest.mark.skipif(not WINDOWS, reason='Not support on none-windows') def test_is_super_path_win(self): assert is_super_path('c:/aa/bb', 'c:/aa/bb\\cc') assert is_super_path('c:/aa/bb', 'c:/aa\\bb/cc') assert is_super_path('c:/aa\\bb', 'c:\\aa/bb/cc') assert is_super_path('c:/', 'c:\\') def test_normjoin(): if WINDOWS: assert normjoin('C:\\', 'b') == 'C:\\b' assert normjoin('C:\\', '\\b') == 'C:\\b' assert normjoin('C:\\a', '\\b') == 'C:\\b' assert normjoin('C:\\a', '..\\b') == 'C:\\b' else: assert normjoin('/a', 'b') == '/a/b' assert normjoin('/a', '/b') == '/b' assert normjoin('/a', '../b') == '/b' def test_filename(): assert filename('/foo/bar') == 'bar' assert filename('/foo/bar.ext') == 'bar' assert filename('/foo/bar.more.ext') == 'bar.more' def test_fileext(): assert fileext('/foo/bar') == '' assert fileext('/foo/bar.ext') == '.ext' assert fileext('/foo/bar.more.ext') == '.ext' ================================================ FILE: tests/test_platform.py ================================================ from pydu.platform import (WINDOWS, LINUX, POSIX, DARWIN, SUNOS, SMARTOS, FREEBSD, NETBSD, OPENBSD, AIX) def test_platform_constants(): assert any([WINDOWS, LINUX, POSIX, DARWIN, SUNOS, SMARTOS, FREEBSD, NETBSD, OPENBSD, AIX]) ================================================ FILE: tests/test_request.py ================================================ import socket from .testing import mockserver import pydu.request from pydu.network import get_free_port from pydu.request import FileName, check_connect, update_query_params, cookies_str_to_dict def test_filename_from_url(): url = 'http://www.example.com/test.txt' assert FileName.from_url(url) == 'test.txt' url = 'http://www.example.com/' assert FileName.from_url(url) is None def test_filename_from_headers(): headers = {'Content-Disposition': 'attachment; filename=test.txt'} assert FileName.from_headers(headers) == 'test.txt' headers = [('Content-Disposition', 'attachment; filename=test.txt')] assert FileName.from_headers(headers) == 'test.txt' headers = 'Content-Disposition: attachment; filename=test.txt' assert FileName.from_headers(headers) == 'test.txt' headers = 'Content-Disposition: attachment; filename=abc/test.txt' assert FileName.from_headers(headers) == 'test.txt' headers = '' assert FileName.from_headers(headers) is None headers = 'Content-Disposition: abc' assert FileName.from_headers(headers) is None headers = 'Content-Disposition: abc;' assert FileName.from_headers(headers) is None headers = 'Content-Disposition: attachment; filename=test.txt; filename=test2.txt' assert FileName.from_headers(headers) is None headers = 'Content-Disposition: attachment; filename=' assert FileName.from_headers(headers) is None @mockserver def test_check_connect(port=None): assert check_connect('127.0.0.1', port=port, timeout=0.01) assert not check_connect('127.0.0.1', port=get_free_port(), timeout=0.01) def mock_socket(*args): raise socket.error old_socket = pydu.request.socket.socket pydu.request.socket.socket = mock_socket assert not check_connect('127.0.0.1', port=port, timeout=0.01) pydu.request.socket.socket = old_socket def test_update_query_params(): base = 'http://example.com/' assert update_query_params(base, {'foo': 1}) == base + '?foo=1' assert update_query_params(base + '?foo=1', {'foo': 2}) == base + '?foo=2' assert update_query_params(base + '?foo=1', {'foo': 2, 'bar': 3}) in \ (base + '?foo=2&bar=3', base + '?bar=3&foo=2') def test_cookies_str_to_dict(): cookies = cookies_str_to_dict('a=a; \tb=b;\nc=c;d;e=') assert cookies['a'] == 'a' assert cookies['b'] == 'b' assert cookies['c'] == 'c' assert cookies['e'] == '' ================================================ FILE: tests/test_set.py ================================================ from pydu.set import OrderedSet def test_ordered_set(): ordered_set = OrderedSet([1, 3, 1, 2]) assert list(ordered_set) == [1, 3, 2] assert 1 in ordered_set assert bool(ordered_set) ordered_set.add(1) assert 1 in ordered_set ordered_set.remove(1) assert 1 not in ordered_set for i in range(4): ordered_set.discard(i) assert not bool(ordered_set) ================================================ FILE: tests/test_slot.py ================================================ from pydu.slot import SlotBase class Foo(SlotBase): __slots__ = ('a', 'b', 'c') class TestSlotBase(object): def test_args(self): foo = Foo(1) assert foo.a == 1 assert foo.b is None assert foo.c is None def test_kwargs(self): foo = Foo(b=2) assert foo.a is None assert foo.b == 2 assert foo.c is None def test_args_kwargs(self): foo = Foo(1, b=2) assert foo.a == 1 assert foo.b == 2 assert foo.c is None ================================================ FILE: tests/test_string.py ================================================ # coding: utf-8 from pydu.string import (safeencode, safeunicode, strips, lstrips, rstrips, common_prefix, common_suffix, sort) def test_safeencode(): assert safeencode('hello') == b'hello' assert safeencode(1) == b'1' assert safeencode(u'中文') == b'\xe4\xb8\xad\xe6\x96\x87' def test_safeunicode(): assert safeunicode('hello') == u'hello' assert safeunicode(1) == u'1' assert safeunicode('中文') == u'中文' assert safeunicode(u'hello') == u'hello' assert safeunicode(u'中文') == u'中文' def test_lstrips(): assert lstrips('foobbar', '') == 'foobbar' assert lstrips('foobar', 'fo') == 'obar' assert lstrips('foofoobar', 'foo') == 'foobar' assert lstrips('foobarbaz', ('foo', 'bar')) == 'baz' assert lstrips('foobarbaz', ('bar', 'foo')) == 'barbaz' def test_rstrips(): assert rstrips('foobbar', '') == 'foobbar' assert rstrips('foobbar', 'bar') == 'foob' assert rstrips('foobarbar', 'bar') == 'foobar' assert rstrips('fozfoobar', ('bar', 'foo')) == 'foz' assert rstrips('fozfoobar', ('foo', 'bar')) == 'fozfoo' def test_strips(): assert strips('foobarfoo', '') == 'foobarfoo' assert strips('foobarfoo', 'foo') == 'bar' assert strips('foobarfoo', ('foo', 'bar')) == '' def test_common_prefix(): l = ['abcd', 'abc1'] assert common_prefix(l) == 'abc' def test_common_suffix(): l = ['dabc', '1abc'] assert common_suffix(l) == 'abc' def test_sort(): assert sort('acb21') == '12abc' assert sort('abc21', reverse=True) == 'cba21' ================================================ FILE: tests/test_system.py ================================================ import os import stat import time import pytest from pydu.platform import WINDOWS from pydu.system import (FileTracker, makedirs, remove, removes, open_file, copy, touch, chmod, which) if not WINDOWS: from pydu.system import link, symlink class TestFileTracker: def test_track_open(self, tmpdir): FileTracker.track() path = tmpdir.join('test').strpath f = open(path, 'w') assert f in FileTracker.get_openfiles() f.close() assert f not in FileTracker.get_openfiles() def test_track_context_open(self, tmpdir): FileTracker.track() path = tmpdir.join('test').strpath with open(path, 'w') as f: assert f in FileTracker.get_openfiles() assert f not in FileTracker.get_openfiles() def test_untrack(self, tmpdir): FileTracker.track() FileTracker.untrack() path = tmpdir.join('test').strpath f = open(path, 'w') assert f not in FileTracker.get_openfiles() class TestMakeDirs: def test_makedirs(self, tmpdir): path = str(tmpdir.join('test')) makedirs(path) assert os.path.exists(path) def test_makedirs_with_exists_path(self, tmpdir): path = str(tmpdir.join('test')) makedirs(path) makedirs(path, exist_ok=True) with pytest.raises(Exception): makedirs(path, exist_ok=False) def test_makedirs_with_ignore_error(self, tmpdir): path = str(tmpdir.join('test')) makedirs(path) makedirs(path, ignore_errors=True) def test_makedirs_without_ignore_error(self, tmpdir): path = str(tmpdir.join('test')) makedirs(path) with pytest.raises(Exception): makedirs(path, ignore_errors=False, exist_ok=False) def test_makedirs_with_mutl_dirs(self, tmpdir): path = str(tmpdir.join('test/test')) makedirs(path) assert os.path.exists(path) def test_touch(tmpdir): path = str(tmpdir.join('test')) touch(path) assert os.path.isfile(path) class TestRemove: def test_remove_dir(self, tmpdir): path = str(tmpdir.join('test')) makedirs(path) remove(path) assert not os.path.exists(path) def test_remove_file(self, tmpdir): f = str(tmpdir.join('test.txt')) touch(f) remove(f) assert not os.path.exists(f) def test_remove_mutil_dirs(self, tmpdir): path = str(tmpdir.join('test/test')) makedirs(path) path = str(tmpdir.join('test')) remove(path) assert not os.path.exists(path) def test_remove_with_ignore_error(self, tmpdir): path = str(tmpdir.join('test')) remove(path, ignore_errors=True) def test_remove_without_ignore_error(self, tmpdir): path = str(tmpdir.join('test')) with pytest.raises(Exception): remove(path, ignore_errors=False) def test_remove_without_ignore_error_with_onerror(self): pass class TestRemoves: def test_removes_paths(self, tmpdir): path1 = str(tmpdir.join('test1')) path2 = str(tmpdir.join('test2')) path3 = str(tmpdir.join('test3')) for path in [path1, path2, path3]: makedirs(path) removes([path1, path2, path3]) assert not os.path.exists(path1) assert not os.path.exists(path2) assert not os.path.exists(path3) def test_removes_files(self, tmpdir): f1 = str(tmpdir.join('test1.txt')) f2 = str(tmpdir.join('test2.txt')) f3 = str(tmpdir.join('test3.txt')) for f in [f1, f2, f3]: touch(f) removes([f1, f2, f3]) for f in [f1, f2, f3]: assert not os.path.exists(f) def test_removes_files_and_path(self, tmpdir): f1 = str(tmpdir.join('test1.txt')) f2 = str(tmpdir.join('test2.txt')) p1 = str(tmpdir.join('test1')) p2 = str(tmpdir.join('test2')) for f in [f1, f2]: touch(f) for p in [p1, p2]: makedirs(p) removes([f1, f2, p1, p2]) for f in [f1, f2, p1, p2]: assert not os.path.exists(f) class TestOpenFile: def test_open_file_without_parent_dir(self, tmpdir): f = str(tmpdir.join('test/test1.txt')) open_file(f) assert os.path.exists(f) def test_open_file_in_exist_path(self, tmpdir): f = str(tmpdir.join('test2.txt')) open_file(f) assert os.path.exists(f) def test_open_exist_file(self, tmpdir): f = str(tmpdir.join('test3.txt')) open_file(f) with open(f, 'r') as f: with pytest.raises(Exception): os.remove(f) def test_open_file_with_ignore_error(self, tmpdir): f = str(tmpdir.join('test1.txt')) open_file(f, mode='r', ignore_errors=True) def test_open_file_without_ignore_error(self, tmpdir): f = str(tmpdir.join('test1.txt')) with pytest.raises(Exception): open_file(f, mode='r') @pytest.mark.skipif(WINDOWS, reason='Not support on windows') class TestLink: def test_link_a_file(self, tmpdir): f = str(tmpdir.join('test.txt')) link_f = str(tmpdir.join('test.link')) touch(f) link(f, link_f) assert os.path.exists(link_f) def test_link_with_ignore_error(self, tmpdir): dirname = str(tmpdir.join('test')) link_dirname = str(tmpdir.join('test.link')) makedirs(dirname) link(dirname, link_dirname, ignore_errors=True) def test_link_without_ignore_error(self, tmpdir): d = str(tmpdir.join('test')) link_d = str(tmpdir.join('test.link')) makedirs(d) with pytest.raises(Exception): link(d, link_d) def test_link_with_overwrite(self, tmpdir): f = str(tmpdir.join('test.txt')) link_f = str(tmpdir.join('test.link')) touch(f) link(f, link_f) t1 = os.path.getctime(link_f) time.sleep(1) link(f, link_f, overwrite=True) t2 = os.path.getctime(link_f) assert t1 != t2 @pytest.mark.skipif(WINDOWS, reason='Not support on windows') class TestSymLink: def test_symlink_a_file(self, tmpdir): f = str(tmpdir.join('test.txt')) link_f = str(tmpdir.join('test.link')) touch(f) symlink(f, link_f) assert os.path.exists(link_f) assert os.path.islink(link_f) def test_symlink_with_ignore_error(self, tmpdir): d = str(tmpdir.join('test')) link_d = str(tmpdir.join('test.link')) makedirs(d) link(d, link_d, ignore_errors=True) def test_symlink_with_overwrite(self, tmpdir): f = str(tmpdir.join('test.txt')) link_f = str(tmpdir.join('test.link')) touch(f) symlink(f, link_f) t1 = os.lstat(link_f).st_ctime time.sleep(1) symlink(f, link_f, overwrite=True) t2 = os.lstat(link_f).st_ctime assert t1 != t2 def test_symlink_without_ignore_error(self, tmpdir): d = str(tmpdir.join('test')) link_d = str(tmpdir.join('test.link')) makedirs(d) with pytest.raises(Exception): link(d, link_d) class TestCopy: def test_copy_file(self, tmpdir): f = str(tmpdir.join('test.txt')) f_copy = str(tmpdir.join('test_copy.txt')) touch(f) copy(f, f_copy) assert os.path.exists(f_copy) def test_copy_non_empty_dir(self, tmpdir): f = str(tmpdir.join('test/test.txt')) d = str(tmpdir.join('test')) d_copy = str(tmpdir.join('test_copy')) os.makedirs(d) touch(f) copy(d, d_copy) assert os.path.exists(d_copy) def test_copy_empty_dir(self, tmpdir): d = str(tmpdir.join('test')) d_copy = str(tmpdir.join('test_copy')) makedirs(d) copy(d, d_copy) assert os.path.exists(d_copy) @pytest.mark.skipif(WINDOWS, reason='Not support on windows') def test_copy_dir_follow_symlink(self, tmpdir): f = str(tmpdir.join('test/test.txt')) d = str(tmpdir.join('test')) link_f = str(tmpdir.join('test/test_link.txt')) d_copy = str(tmpdir.join('test_copy')) new_link_f = str(tmpdir.join('test_copy/test_link.txt')) makedirs(d) touch(f) os.symlink(f, link_f) copy(d, d_copy, follow_symlinks=True) assert os.path.exists(d_copy) assert os.path.islink(new_link_f) @pytest.mark.skipif(WINDOWS, reason='Not support on windows') def test_copy_dir_not_follow_symlink(self, tmpdir): f = str(tmpdir.join('test/test.txt')) d = str(tmpdir.join('test')) link_f = str(tmpdir.join('test/test_link.txt')) d_copy = str(tmpdir.join('test_copy')) new_link_f = str(tmpdir.join('test_copy/test_link.txt')) makedirs(d) touch(f) os.symlink(f, link_f) copy(d, d_copy, follow_symlinks=False) assert os.path.exists(d_copy) assert not os.path.islink(new_link_f) @pytest.mark.skipif(WINDOWS, reason='Not support on windows') def test_copy_file_follow_symlink(self, tmpdir): f = str(tmpdir.join('test.txt')) link_f = str(tmpdir.join('test.link')) copy_link_f = str(tmpdir.join('test_copy.link')) touch(f) link(f, link_f) copy(link_f, copy_link_f, follow_symlinks=True) assert os.path.exists(copy_link_f) assert not os.path.islink(copy_link_f) @pytest.mark.skipif(WINDOWS, reason='Not support on windows') def test_copy_file_not_follow_symlink(self, tmpdir): f = str(tmpdir.join('test.txt')) link_f = str(tmpdir.join('test.link')) copy_link_f = str(tmpdir.join('test_copy.link')) touch(f) os.symlink(f, link_f) copy(link_f, copy_link_f, follow_symlinks=False) assert os.path.exists(copy_link_f) assert os.path.islink(copy_link_f) def test_copy_path_to_exits_path(self, tmpdir): d1 = str(tmpdir.join('test1')) d2 = str(tmpdir.join('test2')) makedirs(d1) makedirs(d2) with pytest.raises(Exception): copy(d1, d2) def test_copy_without_ignore_error(self, tmpdir): d1 = str(tmpdir.join('test1')) d2 = str(tmpdir.join('test2')) makedirs(d1) makedirs(d2) with pytest.raises(Exception): copy(d1, d2, ignore_errors=False) def test_copy_with_ignore_error(self, tmpdir): d1 = str(tmpdir.join('test1')) d2 = str(tmpdir.join('test2')) makedirs(d1) makedirs(d2) copy(d1, d2, ignore_errors=True) @pytest.fixture() def mycmd(tmpdir): mycmd = str(tmpdir.join('mycmd')) touch(mycmd) os.chmod(mycmd, os.F_OK | stat.S_IXUSR) os.environ['PATHEXT'] = '' return mycmd @pytest.mark.usefixtures('mycmd') class TestWhich: def test_dir_cmd(self, mycmd): assert which('noexists/mycmd') is None assert which(mycmd) == mycmd def test_cmd_path(self, tmpdir, mycmd): path = str(tmpdir) assert which('mycmd') is None assert which('mycmd', path=path) == mycmd os.environ['PATH'] = path + os.pathsep + \ os.environ.get('PATH', os.defpath) assert which('mycmd') == mycmd @pytest.mark.skipif(not WINDOWS, reason='Not support non Windows') def test_chcp(): from pydu.system import chcp from ctypes import windll origin_code = windll.kernel32.GetConsoleOutputCP() with chcp(437): assert windll.kernel32.GetConsoleOutputCP() == 437 assert windll.kernel32.GetConsoleOutputCP() == origin_code try: cp = chcp(437) assert windll.kernel32.GetConsoleOutputCP() == 437 assert str(cp) == '' finally: windll.kernel32.SetConsoleOutputCP(origin_code) class TestChmod: def test_chmod_file(self, tmpdir): test_file = tmpdir.join('test_file') touch(test_file.strpath) chmod(test_file.strpath, 0o755) mode = oct(test_file.stat().mode)[-3:] if WINDOWS: assert mode == '666' else: assert mode == '755' chmod(test_file.strpath, 0o444) mode = oct(test_file.stat().mode)[-3:] assert mode == '444' def test_chmod_dir(self, tmpdir): test_dir = tmpdir.mkdir('test_dir') test_sub_dir = test_dir.mkdir('test_dir') test_sub_file = test_dir.join('test_file') touch(test_sub_file.strpath) if WINDOWS: chmod(test_dir.strpath, 0o444, recursive=False) assert oct(test_dir.stat().mode)[-3:] == '555' assert oct(test_sub_dir.stat().mode)[-3:] != '444' assert oct(test_sub_file.stat().mode)[-3:] != '444' chmod(test_dir.strpath, 0o444, recursive=True) assert oct(test_dir.stat().mode)[-3:] == '555' assert oct(test_sub_dir.stat().mode)[-3:] == '555' assert oct(test_sub_file.stat().mode)[-3:] == '444' else: chmod(test_dir.strpath, 0o744, recursive=False) assert oct(test_dir.stat().mode)[-3:] == '744' assert oct(test_sub_dir.stat().mode)[-3:] != '744' assert oct(test_sub_file.stat().mode)[-3:] != '744' chmod(test_dir.strpath, 0o744, recursive=True) assert oct(test_dir.stat().mode)[-3:] == '744' assert oct(test_sub_dir.stat().mode)[-3:] == '744' assert oct(test_sub_file.stat().mode)[-3:] == '744' ================================================ FILE: tests/test_unit.py ================================================ from pydu.unit import Bytes class TestBytes: def test_convert(self): assert Bytes(1024*1024).convert() == (1, 'MB') assert Bytes(1024*1024).convert(unit='KB') == (1024, 'KB') assert Bytes(1000).convert(multiple=1000) == (1, 'KB') ================================================ FILE: tests/testing.py ================================================ import functools from threading import Thread from pydu.network import get_free_port from pydu.inspect import func_supports_parameter from pydu.compat import PY2, ClassTypes if PY2: from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler else: from http.server import HTTPServer as HTTPServer from http.server import BaseHTTPRequestHandler class mockserverfy(object): def __init__(self, RequestHandler=BaseHTTPRequestHandler): self.RequestHandler = RequestHandler self.server = None self.port = None def __enter__(self): self.port = get_free_port() self.server = HTTPServer(('127.0.0.1', self.port), self.RequestHandler) t = Thread(target=self.server.serve_forever) t.setDaemon(True) t.start() return self def __exit__(self, exc_type, exc_value, traceback): self.server.shutdown() def mockserver(test): """A decorator tests that use mock server""" def decorate_class(klass): for attr in dir(klass): if not attr.startswith('test_'): continue attr_value = getattr(klass, attr) if not hasattr(attr_value, "__call__"): continue setattr(klass, attr, decorate_callable(attr_value)) return klass def decorate_callable(test): @functools.wraps(test) def wrapper(*args, **kwargs): with mockserverfy() as server: if func_supports_parameter(test, 'port'): return test(*args, port=server.port, **kwargs) else: return test(*args, **kwargs) return wrapper if isinstance(test, ClassTypes): return decorate_class(test) return decorate_callable(test) ================================================ FILE: tox.ini ================================================ [tox] envlist = py27,py35,py36,py37,py38 [testenv] deps=-rrequirements-dev.txt commands= coverage run -m pytest