Repository: google/grumpy Branch: master Commit: 3ec87959189c Files: 295 Total size: 3.3 MB Directory structure: gitextract_ey0d_gsm/ ├── .gitignore ├── .pylintrc ├── .travis.yml ├── AUTHORS.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── __init__.py ├── benchmarks/ │ ├── bm_call_method.py │ ├── bm_call_simple.py │ ├── call.py │ ├── comprehension.py │ ├── concurrent.py │ ├── dict.py │ ├── generator.py │ ├── list.py │ ├── loop.py │ └── tuple.py ├── compiler/ │ ├── __init__.py │ ├── block.py │ ├── block_test.py │ ├── expr.py │ ├── expr_visitor.py │ ├── expr_visitor_test.py │ ├── imputil.py │ ├── imputil_test.py │ ├── shard_test.py │ ├── stmt.py │ ├── stmt_test.py │ ├── util.py │ └── util_test.py ├── lib/ │ ├── README.md │ ├── __builtin__.py │ ├── _random.py │ ├── _syscall.py │ ├── errno.py │ ├── exceptions.py │ ├── itertools.py │ ├── itertools_test.py │ ├── math.py │ ├── math_test.py │ ├── os/ │ │ ├── __init__.py │ │ ├── path.py │ │ └── path_test.py │ ├── os_test.py │ ├── random_test.py │ ├── select_.py │ ├── stat.py │ ├── sys.py │ ├── sys_test.py │ ├── tempfile.py │ ├── tempfile_test.py │ ├── thread.py │ ├── time.py │ ├── time_test.py │ ├── types_test.py │ ├── weetest.py │ └── weetest_test.py ├── runtime/ │ ├── baseexception.go │ ├── baseexception_test.go │ ├── basestring.go │ ├── basestring_test.go │ ├── bool.go │ ├── bool_test.go │ ├── builtin_types.go │ ├── builtin_types_test.go │ ├── bytearray.go │ ├── bytearray_test.go │ ├── code.go │ ├── code_test.go │ ├── complex.go │ ├── complex_test.go │ ├── core.go │ ├── core_test.go │ ├── descriptor.go │ ├── descriptor_test.go │ ├── dict.go │ ├── dict_test.go │ ├── doc.go │ ├── exceptions.go │ ├── file.go │ ├── file_test.go │ ├── float.go │ ├── float_test.go │ ├── frame.go │ ├── frame_test.go │ ├── function.go │ ├── function_test.go │ ├── generator.go │ ├── generator_test.go │ ├── int.go │ ├── int_test.go │ ├── list.go │ ├── list_test.go │ ├── long.go │ ├── long_test.go │ ├── method.go │ ├── method_test.go │ ├── module.go │ ├── module_test.go │ ├── native.go │ ├── native_test.go │ ├── numeric.go │ ├── object.go │ ├── object_test.go │ ├── param.go │ ├── param_test.go │ ├── range.go │ ├── range_test.go │ ├── seq.go │ ├── seq_test.go │ ├── set.go │ ├── set_test.go │ ├── slice.go │ ├── slice_test.go │ ├── slots.go │ ├── slots_test.go │ ├── str.go │ ├── str_test.go │ ├── super.go │ ├── super_test.go │ ├── threading.go │ ├── threading_test.go │ ├── traceback.go │ ├── tuple.go │ ├── tuple_test.go │ ├── type.go │ ├── type_test.go │ ├── unicode.go │ ├── unicode_test.go │ ├── weakref.go │ └── weakref_test.go ├── testing/ │ ├── assert_test.py │ ├── assign_test.py │ ├── builtin_test.py │ ├── class_test.py │ ├── compare_test.py │ ├── complex_test.py │ ├── comprehension_test.py │ ├── dict_test.py │ ├── file_test.py │ ├── float_test.py │ ├── for_test.py │ ├── function_test.py │ ├── generator_test.py │ ├── getopt_test.py │ ├── global_test.py │ ├── if_test.py │ ├── import_test.py │ ├── list_test.py │ ├── native_test.py │ ├── op_test.py │ ├── pow_test.py │ ├── scope_test.py │ ├── str_test.py │ ├── struct_test.py │ ├── try_test.py │ ├── tuple_test.py │ ├── while_test.py │ └── with_test.py ├── third_party/ │ ├── ouroboros/ │ │ ├── AUTHORS │ │ ├── LICENSE │ │ ├── READEME.md │ │ ├── operator.py │ │ └── test/ │ │ └── test_operator.py │ ├── pypy/ │ │ ├── LICENSE │ │ ├── README.md │ │ ├── _collections.py │ │ ├── _csv.py │ │ ├── _functools.py │ │ ├── _md5.py │ │ ├── _sha.py │ │ ├── _sha256.py │ │ ├── _sha512.py │ │ ├── _sre.py │ │ ├── _struct.py │ │ ├── binascii.py │ │ └── datetime.py │ ├── pythonparser/ │ │ ├── LICENSE.txt │ │ ├── README.md │ │ ├── __init__.py │ │ ├── algorithm.py │ │ ├── ast.py │ │ ├── diagnostic.py │ │ ├── lexer.py │ │ ├── parser.py │ │ └── source.py │ └── stdlib/ │ ├── LICENSE │ ├── Queue.py │ ├── README.md │ ├── StringIO.py │ ├── UserDict.py │ ├── UserList.py │ ├── UserString.py │ ├── _abcoll.py │ ├── _weakrefset.py │ ├── abc.py │ ├── argparse.py │ ├── base64.py │ ├── bisect.py │ ├── collections.py │ ├── colorsys.py │ ├── contextlib.py │ ├── copy.py │ ├── copy_reg.py │ ├── csv.py │ ├── difflib.py │ ├── dircache.py │ ├── dummy_thread.py │ ├── fnmatch.py │ ├── fpformat.py │ ├── functools.py │ ├── genericpath.py │ ├── getopt.py │ ├── glob.py │ ├── heapq.py │ ├── json/ │ │ ├── __init__.py │ │ ├── decoder.py │ │ └── encoder.py │ ├── json_scanner.py │ ├── keyword.py │ ├── linecache.py │ ├── md5.py │ ├── mimetools.py │ ├── mutex.py │ ├── optparse.py │ ├── pprint.py │ ├── quopri.py │ ├── random.py │ ├── re.py │ ├── re_tests.py │ ├── repr.py │ ├── rfc822.py │ ├── sched.py │ ├── sha.py │ ├── sre_compile.py │ ├── sre_constants.py │ ├── sre_parse.py │ ├── string.py │ ├── test/ │ │ ├── __init__.py │ │ ├── list_tests.py │ │ ├── lock_tests.py │ │ ├── mapping_tests.py │ │ ├── seq_tests.py │ │ ├── string_tests.py │ │ ├── test_argparse.py │ │ ├── test_bisect.py │ │ ├── test_colorsys.py │ │ ├── test_datetime.py │ │ ├── test_dict.py │ │ ├── test_dircache.py │ │ ├── test_dummy_thread.py │ │ ├── test_fpformat.py │ │ ├── test_genericpath.py │ │ ├── test_list.py │ │ ├── test_md5.py │ │ ├── test_mimetools.py │ │ ├── test_mutex.py │ │ ├── test_queue.py │ │ ├── test_quopri.py │ │ ├── test_rfc822.py │ │ ├── test_sched.py │ │ ├── test_select.py │ │ ├── test_slice.py │ │ ├── test_stat.py │ │ ├── test_string.py │ │ ├── test_support.py │ │ ├── test_threading.py │ │ ├── test_tuple.py │ │ └── test_uu.py │ ├── textwrap.py │ ├── threading.py │ ├── traceback.py │ ├── types.py │ ├── unittest/ │ │ └── __init__.py │ ├── unittest_case.py │ ├── unittest_loader.py │ ├── unittest_result.py │ ├── unittest_runner.py │ ├── unittest_signals.py │ ├── unittest_suite.py │ ├── unittest_util.py │ ├── urlparse.py │ ├── uu.py │ ├── warnings.py │ └── weakref.py └── tools/ ├── benchcmp ├── coverparse ├── diffrange ├── genmake ├── grumpc ├── grumprun ├── pkgc.go └── pydeps ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ build errors.err *.swp *.pyc ================================================ FILE: .pylintrc ================================================ [BASIC] argument-rgx=^[a-z][a-z0-9_]*$ attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ docstring-min-length=10 function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ good-names=main,_ method-rgx=^(?:(?P__[a-z0-9_]+__|next|test[A-Z_][A-Za-z0-9_]*)|(?P_{0,2}[A-Z][a-zA-Z0-9]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ no-docstring-rgx=(__.*__|main|test[A-Z_][A-Za-z0-9_]*|[A-Z][A-Za-z0-9]*Test) variable-rgx=^[a-z][a-z0-9_]*$ [FORMAT] indent-string=' ' max-line-length=80 [MESSAGES CONTROL] # TODO: Remove cyclic-import once expr_visitor.py -> stmt.py is resolved. disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-disabled,star-args,pointless-except,bad-option-value,global-statement,fixme,suppressed-message,useless-suppression,locally-enabled,file-ignored,cyclic-import [REPORTS] msg-template={path}:{line}: {msg} ({symbol}) reports=no [TYPECHECK] # AST classes have dynamic members. Writer does not but for some reason pylint # barfs on some of its members. ignored-classes=pythonparser.ast.Module,grumpy.compiler.util.Writer ================================================ FILE: .travis.yml ================================================ language: go os: - linux - osx before_script: - rvm get head || true # https://github.com/travis-ci/travis-ci/issues/6307 - set -e # Run gofmt and lint serially to avoid confusing output. Run tests in parallel # for speed. script: make gofmt lint && make -j2 test after_script: set +e ================================================ FILE: AUTHORS.md ================================================ Contributors in the order of first contribution * [Dylan Trotter](https://github.com/trotterdylan) * [Aaron Tubbs](https://github.com/atubbs) * [Devin Jeanpierre](https://github.com/ssbr) * [Torin Rudeen](https://github.com/torinmr) * [Brian Atkinson](https://github.com/nairb774) * [Matt Harden](https://github.com/nerdatmath) * [Ryan Kennedy](https://github.com/ryankennedy) * [Brett Cannon](https://github.com/brettcannon) * [Alex Koren](https://github.com/AlexEKoren) * [Steven Maude](https://github.com/StevenMaude) * [feilengcui008](https://github.com/feilengcui008) * [Meador Inge](https://github.com/meadori) * [Kyoung-chan Lee](https://github.com/leekchan) * [Cholerae Hu](https://github.com/choleraehyq) * [Jurriaan Bremer](https://github.com/jbremer) * [Florian Ludwig](https://github.com/FlorianLudwig) * [ns-cweber](https://github.com/ns-cweber) * [Paul Smith](https://github.com/paulsmith) * [Izaak Weiss](https://github.com/lalaithion) * [YOU](https://github.com/S-YOU) * [iury](https://github.com/IuryAlves) * [wuttem](https://github.com/wuttem) * [cclauss](https://github.com/cclauss) * [Mirko Dziadzka](https://github.com/MirkoDziadzka) * [Dong-hee Na](https://github.com/corona10) ================================================ FILE: CONTRIBUTING.md ================================================ Want to contribute? Great! First, read this page. ### Before you contribute Before we can use your code, you must sign the [Google Individual Contributor License Agreement] (https://cla.developers.google.com/about/google-individual) (CLA), which you can do online. The CLA is necessary mainly because you own the copyright to your changes, even after your contribution becomes part of our codebase, so we need your permission to use and distribute your code. We also need to be sure of various other things—for instance that you'll tell us if you know that your code infringes on other people's patents. You don't have to sign the CLA until after you've submitted your code for review and a member has approved it, but you must do it before we can put your code into our codebase. Before you start working on a larger contribution, you should get in touch with us first through the issue tracker with your idea so that we can help out and possibly guide you. Coordinating up front makes it much easier to avoid frustration later on. Contributions made by corporations are covered by a different agreement than the one above, the [Software Grant and Corporate Contributor License Agreement] (https://cla.developers.google.com/about/google-corporate). ### Code reviews All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. ### Code Style We use Google style guides for all our code. Below are the guidelines for each language we use. #### Go All Go source code must be formatted using gofmt and be lint-clean according to golint. This will be checked by Travis but can be checked manually from a local repo via `make gofmt golint`. Code is expected to be gofmt- and lint clean before it is submitted for review. Code reviews can then focus on structural details and higher level style considerations. Many common mistakes are already documented in the [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments) doc so it's worth being familiar with these patterns. #### Python All Python source code must be lint-clean according to pylint. This will be checked by Travis but can be checked manually from a local repo via `make pylint`. Once code is pylint-clean, it can be submitted for review. In addition to lint cleanliness, Python code must adhere to the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html) which has a number of additional conventions. Please be familiar with the style guide and ensure code satisfies its rules before submitting for review. ##### Borrowed Standard Library Code Standard library code that is borrowed from other open source projects such as CPython and PyPy need not be lint clean or satisfy the style guide. The goal should be to keep the copied sources as close to the originals as possible while still being functional. For more details about borrowing this kind of code from other places see the [guidelines for integration](https://github.com/google/grumpy/wiki/Standard-libraries:-guidelines-for-integration). ================================================ FILE: LICENSE ================================================ Copyright 2016 Google Inc. All Rights Reserved. Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2016 Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------ # General setup # ------------------------------------------------------------------------------ GO_ENV := $(shell go env GOOS GOARCH) GOOS ?= $(word 1,$(GO_ENV)) GOARCH ?= $(word 2,$(GO_ENV)) ROOT_DIR := $(realpath .) PKG_DIR := build/pkg/$(GOOS)_$(GOARCH) # Try python2 and then python if PYTHON has not been set ifeq ($(PYTHON),) ifneq ($(shell which python2),) PYTHON = python2 else PYTHON = python endif endif PYTHON_BIN := $(shell which $(PYTHON)) PYTHON_VER := $(word 2,$(shell $(PYTHON) -V 2>&1)) GO_REQ_MAJ := 1 GO_REQ_MIN := 9 GO_MAJ_MIN := $(subst go,, $(word 3,$(shell go version 2>&1)) ) GO_MAJ := $(word 1,$(subst ., ,$(GO_MAJ_MIN) )) GO_MIN := $(word 2,$(subst ., ,$(GO_MAJ_MIN) )) ifeq ($(filter 2.7.%,$(PYTHON_VER)),) $(error unsupported Python version $(PYTHON_VER), Grumpy only supports 2.7.x. To use a different python binary such as python2, run: 'make PYTHON=python2 ...') endif ifneq ($(shell test $(GO_MAJ) -ge $(GO_REQ_MAJ) -a $(GO_MIN) -ge $(GO_REQ_MIN) && echo ok),ok) $(error unsupported Go version $(GO_VER), Grumpy requires at least $(GO_REQ_MAJ).$(GO_REQ_MIN). Please update Go) endif PY_DIR := build/lib/python2.7/site-packages PY_INSTALL_DIR := $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") export GOPATH := $(ROOT_DIR)/build export PYTHONPATH := $(ROOT_DIR)/$(PY_DIR) export PATH := $(ROOT_DIR)/build/bin:$(PATH) GOPATH_PY_ROOT := $(GOPATH)/src/__python__ PYTHONPARSER_SRCS := $(patsubst third_party/%,$(PY_DIR)/grumpy/%,$(wildcard third_party/pythonparser/*.py)) COMPILER_BIN := build/bin/grumpc COMPILER_SRCS := $(addprefix $(PY_DIR)/grumpy/compiler/,$(notdir $(shell find compiler -name '*.py' -not -name '*_test.py'))) $(PY_DIR)/grumpy/__init__.py COMPILER_TESTS := $(patsubst %.py,grumpy/%,$(filter-out compiler/expr_visitor_test.py compiler/stmt_test.py,$(wildcard compiler/*_test.py))) COMPILER_TEST_SRCS := $(patsubst %,$(PY_DIR)/%.py,$(COMPILER_TESTS)) COMPILER_SHARDED_TEST_SRCS := $(patsubst %,$(PY_DIR)/grumpy/compiler/%,expr_visitor_test.py stmt_test.py) COMPILER_PASS_FILES := $(patsubst %,$(PY_DIR)/%.pass,$(COMPILER_TESTS)) COMPILER_EXPR_VISITOR_PASS_FILES := $(patsubst %,$(PY_DIR)/grumpy/compiler/expr_visitor_test.%of32.pass,$(shell seq 32)) COMPILER_STMT_PASS_FILES := $(patsubst %,$(PY_DIR)/grumpy/compiler/stmt_test.%of16.pass,$(shell seq 16)) COMPILER_D_FILES := $(patsubst %,$(PY_DIR)/%.d,$(COMPILER_TESTS)) COMPILER := $(COMPILER_BIN) $(COMPILER_SRCS) $(PYTHONPARSER_SRCS) PKGC_BIN := build/bin/pkgc RUNNER_BIN := build/bin/grumprun RUNTIME_SRCS := $(addprefix build/src/grumpy/,$(notdir $(wildcard runtime/*.go))) RUNTIME := $(PKG_DIR)/grumpy.a RUNTIME_PASS_FILE := build/runtime.pass RUNTIME_COVER_FILE := $(PKG_DIR)/grumpy.cover RUNNER = $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(STDLIB) LIB_SRCS := $(patsubst lib/%,$(GOPATH_PY_ROOT)/%,$(shell find lib -name '*.py')) THIRD_PARTY_STDLIB_SRCS := $(patsubst third_party/stdlib/%,$(GOPATH_PY_ROOT)/%,$(shell find third_party/stdlib -name '*.py')) THIRD_PARTY_PYPY_SRCS := $(patsubst third_party/pypy/%,$(GOPATH_PY_ROOT)/%,$(shell find third_party/pypy -name '*.py')) THIRD_PARTY_OUROBOROS_SRCS := $(patsubst third_party/ouroboros/%,$(GOPATH_PY_ROOT)/%,$(shell find third_party/ouroboros -name '*.py')) STDLIB_SRCS := $(LIB_SRCS) $(THIRD_PARTY_STDLIB_SRCS) $(THIRD_PARTY_PYPY_SRCS) $(THIRD_PARTY_OUROBOROS_SRCS) STDLIB_PACKAGES := $(patsubst $(GOPATH_PY_ROOT)/%.py,%,$(patsubst $(GOPATH_PY_ROOT)/%/__init__.py,%,$(STDLIB_SRCS))) STDLIB := $(patsubst %,$(PKG_DIR)/__python__/%.a,$(STDLIB_PACKAGES)) STDLIB_TESTS := \ itertools_test \ math_test \ os/path_test \ os_test \ random_test \ re_tests \ sys_test \ tempfile_test \ test/test_bisect \ test/test_colorsys \ test/test_datetime \ test/test_dict \ test/test_dircache \ test/test_dummy_thread \ test/test_fpformat \ test/test_genericpath \ test/test_list \ test/test_md5 \ test/test_mimetools \ test/test_mutex \ test/test_operator \ test/test_quopri \ test/test_queue \ test/test_rfc822 \ test/test_sched \ test/test_select \ test/test_slice \ test/test_stat \ test/test_string \ test/test_threading \ test/test_tuple \ test/test_uu \ time_test \ types_test \ weetest_test STDLIB_PASS_FILES := $(patsubst %,build/testing/%.pass,$(notdir $(STDLIB_TESTS))) ACCEPT_TESTS := $(patsubst %.py,%,$(wildcard testing/*.py)) ACCEPT_PASS_FILES := $(patsubst %,build/%.pass,$(ACCEPT_TESTS)) ACCEPT_PY_PASS_FILES := $(patsubst %,build/%_py.pass,$(filter-out %/native_test,$(ACCEPT_TESTS))) BENCHMARKS := $(patsubst %.py,%,$(wildcard benchmarks/*.py)) BENCHMARK_BINS := $(patsubst %,build/%_benchmark,$(BENCHMARKS)) TOOL_BINS = $(patsubst %,build/bin/%,benchcmp coverparse diffrange genmake pydeps) GOLINT_BIN = build/bin/golint PYLINT_BIN = build/bin/pylint all: $(COMPILER) $(RUNNER) $(RUNTIME) $(TOOL_BINS) benchmarks: $(BENCHMARK_BINS) clean: @rm -rf build # Convenient wrapper around grumprun that avoids having to set up PATH, etc. run: $(RUNNER) @$(RUNNER_BIN) test: $(ACCEPT_PASS_FILES) $(ACCEPT_PY_PASS_FILES) $(COMPILER_PASS_FILES) $(COMPILER_EXPR_VISITOR_PASS_FILES) $(COMPILER_STMT_PASS_FILES) $(RUNTIME_PASS_FILE) $(STDLIB_PASS_FILES) precommit: cover gofmt lint test .PHONY: all benchmarks clean cover gofmt golint install lint precommit pylint run test # ------------------------------------------------------------------------------ # grumpc compiler # ------------------------------------------------------------------------------ $(COMPILER_BIN) $(RUNNER_BIN) $(TOOL_BINS): build/bin/%: tools/% @mkdir -p build/bin @cp -f $< $@ @sed -i.bak -e '1s@/usr/bin/env python@$(PYTHON_BIN)@' $@ @rm -f $@.bak $(COMPILER_SRCS) $(COMPILER_TEST_SRCS) $(COMPILER_SHARDED_TEST_SRCS): $(PY_DIR)/grumpy/%.py: %.py @mkdir -p $(PY_DIR)/grumpy/compiler @cp -f $< $@ $(COMPILER_PASS_FILES): %.pass: %.py $(COMPILER) $(COMPILER_TEST_SRCS) @$(PYTHON) $< -q @touch $@ @echo compiler/`basename $*` PASS # NOTE: In the regex below we use (\.|$) instead of \> because the latter is # not available in the awk available on OS X. $(COMPILER_D_FILES): $(PY_DIR)/%.d: $(PY_DIR)/%.py $(COMPILER_SRCS) $(PYTHONPARSER_SRCS) @$(PYTHON) -m modulefinder $< | awk '{if (match($$2, /^grumpy(\.|$$)/)) { print "$(PY_DIR)/$*.pass: " substr($$3, length("$(ROOT_DIR)/") + 1) }}' > $@ -include $(COMPILER_D_FILES) # Does not depend on stdlibs since it makes minimal use of them. $(COMPILER_EXPR_VISITOR_PASS_FILES): $(PY_DIR)/grumpy/compiler/expr_visitor_test.%.pass: $(PY_DIR)/grumpy/compiler/expr_visitor_test.py $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(PKG_DIR)/__python__/traceback.a @$(PYTHON) $< --shard=$* @touch $@ @echo 'compiler/expr_visitor_test $* PASS' COMPILER_STMT_PASS_FILE_DEPS := \ $(PKG_DIR)/__python__/__go__/grumpy.a \ $(PKG_DIR)/__python__/__go__/os.a \ $(PKG_DIR)/__python__/__go__/runtime.a \ $(PKG_DIR)/__python__/__go__/time.a \ $(PKG_DIR)/__python__/__go__/unicode.a \ $(PKG_DIR)/__python__/sys.a \ $(PKG_DIR)/__python__/traceback.a # Does not depend on stdlibs since it makes minimal use of them. $(COMPILER_STMT_PASS_FILES): $(PY_DIR)/grumpy/compiler/stmt_test.%.pass: $(PY_DIR)/grumpy/compiler/stmt_test.py $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(COMPILER_STMT_PASS_FILE_DEPS) @$(PYTHON) $< --shard=$* @touch $@ @echo 'compiler/stmt_test $* PASS' $(PKGC_BIN): tools/pkgc.go @mkdir -p $(@D) @go build -o $@ $< # ------------------------------------------------------------------------------ # Grumpy runtime # ------------------------------------------------------------------------------ $(RUNTIME_SRCS): build/src/grumpy/%.go: runtime/%.go @mkdir -p build/src/grumpy @cp -f $< $@ $(RUNTIME): $(filter-out %_test.go,$(RUNTIME_SRCS)) @mkdir -p $(PKG_DIR) @go tool compile -o $@ -p grumpy -complete -I $(PKG_DIR) -pack $^ $(RUNTIME_PASS_FILE): $(RUNTIME) $(filter %_test.go,$(RUNTIME_SRCS)) @go test grumpy @touch $@ @echo 'runtime/grumpy PASS' $(RUNTIME_COVER_FILE): $(RUNTIME) $(filter %_test.go,$(RUNTIME_SRCS)) @go test -coverprofile=$@ grumpy cover: $(RUNTIME_COVER_FILE) $(TOOL_BINS) @bash -c 'comm -12 <(coverparse $< | sed "s/^grumpy/runtime/" | sort) <(git diff --dst-prefix= $(DIFF_COMMIT) | diffrange | sort)' | sort -t':' -k1,1 -k2n,2 | sed 's/$$/: missing coverage/' | tee errors.err @test ! -s errors.err build/gofmt.diff: $(wildcard runtime/*.go) @gofmt -d $^ > $@ gofmt: build/gofmt.diff @if [ -s $< ]; then echo 'gofmt found errors, run: gofmt -w $(ROOT_DIR)/runtime/*.go'; false; fi $(GOLINT_BIN): @go get -u github.com/golang/lint/golint golint: $(GOLINT_BIN) @$(GOLINT_BIN) -set_exit_status runtime # Don't use system pip for this since behavior varies a lot between versions. # Instead build pylint from source. Dependencies will be fetched by distutils. $(PYLINT_BIN): @mkdir -p build/third_party @cd build/third_party && curl -sL https://pypi.io/packages/source/p/pylint/pylint-1.6.4.tar.gz | tar -zx @cd build/third_party/pylint-1.6.4 && $(PYTHON) setup.py install --prefix $(ROOT_DIR)/build pylint: $(PYLINT_BIN) $(COMPILER_SRCS) $(PYTHONPARSER_SRCS) $(COMPILER_BIN) $(RUNNER_BIN) $(TOOL_BINS) @$(PYTHON) $(PYLINT_BIN) $(COMPILER_SRCS) $(COMPILER_BIN) $(RUNNER_BIN) $(TOOL_BINS) lint: golint pylint # ------------------------------------------------------------------------------ # Native modules # ------------------------------------------------------------------------------ $(PKG_DIR)/__python__/__go__/%.a: build/src/__python__/__go__/%/module.go $(RUNTIME) @mkdir -p $(@D) @go install __python__/__go__/$* build/src/__python__/__go__/%/module.go: $(PKGC_BIN) $(RUNTIME) @mkdir -p $(@D) @$(PKGC_BIN) $* > $@ $(PKG_DIR)/__python__/__go__/grumpy.a: $(RUNTIME) .PRECIOUS: build/src/__python__/__go__/%/module.go $(PKG_DIR)/__python__/__go__/%.a # ------------------------------------------------------------------------------ # Standard library # ------------------------------------------------------------------------------ $(LIB_SRCS): $(GOPATH_PY_ROOT)/%: lib/% @mkdir -p $(@D) @cp -f $< $@ $(THIRD_PARTY_STDLIB_SRCS): $(GOPATH_PY_ROOT)/%: third_party/stdlib/% @mkdir -p $(@D) @cp -f $< $@ $(THIRD_PARTY_PYPY_SRCS): $(GOPATH_PY_ROOT)/%: third_party/pypy/% @mkdir -p $(@D) @cp -f $< $@ $(THIRD_PARTY_OUROBOROS_SRCS): $(GOPATH_PY_ROOT)/%: third_party/ouroboros/% @mkdir -p $(@D) @cp -f $< $@ build/stdlib.mk: build/bin/genmake | $(STDLIB_SRCS) @genmake build > $@ -include build/stdlib.mk $(patsubst %,build/src/__python__/%/module.go,$(STDLIB_PACKAGES)): $(COMPILER) $(patsubst %,build/src/__python__/%/module.d,$(STDLIB_PACKAGES)): build/bin/pydeps $(PYTHONPARSER_SRCS) $(COMPILER) $(patsubst %,$(PKG_DIR)/__python__/%.a,$(STDLIB_PACKAGES)): $(RUNTIME) define GRUMPY_STDLIB_TEST build/testing/$(notdir $(1)).pass: $(RUNTIME) $(PKG_DIR)/__python__/$(1).a $(RUNNER_BIN) $(PKG_DIR)/__python__/traceback.a @mkdir -p $$(@D) @$(RUNNER_BIN) -m $(subst /,.,$(1)) @touch $$@ @echo 'lib/$(1) PASS' endef $(eval $(foreach x,$(STDLIB_TESTS),$(call GRUMPY_STDLIB_TEST,$(x)))) # ------------------------------------------------------------------------------ # Acceptance tests & benchmarks # ------------------------------------------------------------------------------ $(PY_DIR)/weetest.py: lib/weetest.py @cp -f $< $@ $(PYTHONPARSER_SRCS): $(PY_DIR)/grumpy/%: third_party/% @mkdir -p $(@D) @cp -f $< $@ $(ACCEPT_PASS_FILES): build/%_test.pass: %_test.py $(RUNTIME) $(STDLIB) $(RUNNER_BIN) @mkdir -p $(@D) @$(RUNNER_BIN) < $< @touch $@ @echo '$*_test PASS' NATIVE_TEST_DEPS := \ $(PKG_DIR)/__python__/__go__/encoding/csv.a \ $(PKG_DIR)/__python__/__go__/image.a \ $(PKG_DIR)/__python__/__go__/math.a \ $(PKG_DIR)/__python__/__go__/strings.a build/testing/native_test.pass: $(NATIVE_TEST_DEPS) $(ACCEPT_PY_PASS_FILES): build/%_py.pass: %.py $(PY_DIR)/weetest.py @mkdir -p $(@D) @$(PYTHON) $< @touch $@ @echo '$*_py PASS' $(patsubst %,build/%.go,$(BENCHMARKS)): build/%.go: %.py $(COMPILER) @mkdir -p $(@D) @$(COMPILER_BIN) $< > $@ $(BENCHMARK_BINS): build/benchmarks/%_benchmark: build/benchmarks/%.go $(RUNTIME) $(STDLIB) @mkdir -p $(@D) @go build -o $@ $< # ------------------------------------------------------------------------------ # Installation # ------------------------------------------------------------------------------ install: $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(STDLIB) # Binary executables install -d "$(DESTDIR)/usr/bin" install -m755 build/bin/grumpc "$(DESTDIR)/usr/bin/grumpc" install -m755 build/bin/grumprun "$(DESTDIR)/usr/bin/grumprun" # Python module install -d "$(DESTDIR)"{/usr/lib/go,"$(PY_INSTALL_DIR)"} cp -rv "$(PY_DIR)/grumpy" "$(DESTDIR)$(PY_INSTALL_DIR)" # Go package and sources install -d "$(DESTDIR)/usr/lib/go/" cp -rv build/pkg build/src "$(DESTDIR)/usr/lib/go/" ================================================ FILE: README.md ================================================ # Grumpy: Go running Python [![Build Status](https://travis-ci.org/google/grumpy.svg?branch=master)](https://travis-ci.org/google/grumpy) [![Join the chat at https://gitter.im/grumpy-devel/Lobby](https://badges.gitter.im/grumpy-devel/Lobby.svg)](https://gitter.im/grumpy-devel/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) ## Overview Grumpy is a Python to Go source code transcompiler and runtime that is intended to be a near drop-in replacement for CPython 2.7. The key difference is that it compiles Python source code to Go source code which is then compiled to native code, rather than to bytecode. This means that Grumpy has no VM. The compiled Go source code is a series of calls to the Grumpy runtime, a Go library serving a similar purpose to the Python C API (although the API is incompatible with CPython's). ## Limitations ### Things that will probably never be supported by Grumpy 1. `exec`, `eval` and `compile`: These dynamic features of CPython are not supported by Grumpy because Grumpy modules consist of statically-compiled Go code. Supporting dynamic execution would require bundling Grumpy programs with the compilation toolchain, which would be unwieldy and impractically slow. 2. C extension modules: Grumpy has a different API and object layout than CPython and so supporting C extensions would be difficult. In principle it's possible to support them via an API bridge layer like the one that [JyNI](http://jyni.org) provides for Jython, but it would be hard to maintain and would add significant overhead when calling into and out of extension modules. ### Things that Grumpy will support but doesn't yet There are three basic categories of incomplete functionality: 1. [Language features](https://github.com/google/grumpy/wiki/Missing-features#language-features): Most language features are implemented with the notable exception of [old-style classes](http://stackoverflow.com/questions/54867/what-is-the-difference-between-old-style-and-new-style-classes-in-python). There are also a handful of operators that aren't yet supported. 2. [Builtin functions and types](https://github.com/google/grumpy/wiki/Missing-features#builtins): There are a number of missing functions and types in `__builtins__` that have not yet been implemented. There are also a lot of methods on builtin types that are missing. 3. [Standard library](https://github.com/google/grumpy/wiki/Missing-features#standard-libraries): The Python standard library is very large and much of it is pure Python, so as the language features and builtins get filled out, many modules will just work. But there are also a number of libraries in CPython that are C extension modules which will need to be rewritten. 4. C locale support: Go doesn't support locales in the same way that C does. As such, some functionality that is locale-dependent may not currently work the same as in CPython. ## Running Grumpy Programs ### Method 1: make run: The simplest way to execute a Grumpy program is to use `make run`, which wraps a shell script called grumprun that takes Python code on stdin and builds and runs the code under Grumpy. All of the commands below are assumed to be run from the root directory of the Grumpy source code distribution: ``` echo "print 'hello, world'" | make run ``` ### Method 2: grumpc and grumprun: For more complicated programs, you'll want to compile your Python source code to Go using grumpc (the Grumpy compiler) and then build the Go code using `go build`. Since Grumpy programs are statically linked, all the modules in a program must be findable by the Grumpy toolchain on the GOPATH. Grumpy looks for Go packages corresponding to Python modules in the \_\_python\_\_ subdirectory of the GOPATH. By convention, this subdirectory is also used for staging Python source code, making it similar to the PYTHONPATH. The first step is to set up the shell so that the Grumpy toolchain and libraries can be found. From the root directory of the Grumpy source distribution run: ``` make export PATH=$PWD/build/bin:$PATH export GOPATH=$PWD/build export PYTHONPATH=$PWD/build/lib/python2.7/site-packages ``` You will know things are working if you see the expected output from this command: ``` echo 'import sys; print sys.version' | grumprun ``` Next, we will write our simple Python module into the \_\_python\_\_ directory: ``` echo 'def hello(): print "hello, world"' > $GOPATH/src/__python__/hello.py ``` To build a Go package from our Python script, run the following: ``` mkdir -p $GOPATH/src/__python__/hello grumpc -modname=hello $GOPATH/src/__python__/hello.py > \ $GOPATH/src/__python__/hello/module.go ``` You should now be able to build a Go program that imports the package "\_\_python\_\_/hello". We can also import this module into Python programs that are built using grumprun: ``` echo 'from hello import hello; hello()' | grumprun ``` grumprun is doing a few things under the hood here: 1. Compiles the given Python code to a dummy Go package, the same way we produced \_\_python\_\_/hello/module.go above 2. Produces a main Go package that imports the Go package from step 1. and executes it as our \_\_main\_\_ Python package 3. Executes `go run` on the main package generated in step 2. ## Developing Grumpy There are three main components and depending on what kind of feature you're writing, you may need to change one or more of these. ### grumpc Grumpy converts Python programs into Go programs and `grumpc` is the tool responsible for parsing Python code and generating Go code from it. `grumpc` is written in Python and uses the [`pythonparser`](https://github.com/m-labs/pythonparser) module to accomplish parsing. The grumpc script itself lives at `tools/grumpc`. It is supported by a number of Python modules in the `compiler` subdir. ### Grumpy Runtime The Go code generated by `grumpc` performs operations on data structures that represent Python objects in running Grumpy programs. These data structures and operations are defined in the `grumpy` Go library (source is in the runtime subdir of the source distribution). This runtime is analogous to the Python C API and many of the structures and operations defined by `grumpy` have counterparts in CPython. ### Grumpy Standard Library Much of the Python standard library is written in Python and thus "just works" in Grumpy. These parts of the standard library are copied from CPython 2.7 (possibly with light modifications). For licensing reasons, these files are kept in the `third_party` subdir. The parts of the standard library that cannot be written in pure Python, e.g. file and directory operations, are kept in the `lib` subdir. In CPython these kinds of modules are written as C extensions. In Grumpy they are written in Python but they use native Go extensions to access facilities not otherwise available in Python. ### Source Code Overview - `compiler`: Python package implementating Python -> Go transcompilation logic. - `lib`: Grumpy-specific Python standard library implementation. - `runtime`: Go source code for the Grumpy runtime library. - `third_party/ouroboros`: Pure Python standard libraries copied from the [Ouroboros project](https://github.com/pybee/ouroboros). - `third_party/pypy`: Pure Python standard libraries copied from PyPy. - `third_party/stdlib`: Pure Python standard libraries copied from CPython. - `tools`: Transcompilation and utility binaries. ## Contact Questions? Comments? Drop us a line at [grumpy-users@googlegroups.com](https://groups.google.com/forum/#!forum/grumpy-users) or join our [Gitter channel](https://gitter.im/grumpy-devel/Lobby) ================================================ FILE: __init__.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: benchmarks/bm_call_method.py ================================================ """Microbenchmark for method call overhead. This measures simple method calls that are predictable, do not use varargs or kwargs, and do not use tuple unpacking. Taken from: https://github.com/python/performance/blob/9b8d859/performance/benchmarks/bm_call_method.py """ import weetest class Foo(object): def foo(self, a, b, c, d): # 20 calls self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) self.bar(a, b, c) def bar(self, a, b, c): # 20 calls self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) self.baz(a, b) def baz(self, a, b): # 20 calls self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) self.quux(a) def quux(self, a): # 20 calls self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() self.qux() def qux(self): pass def BenchmarkCallMethod(b): f = Foo() for _ in xrange(b.N): # 20 calls f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) f.foo(1, 2, 3, 4) if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: benchmarks/bm_call_simple.py ================================================ """Microbenchmark for function call overhead. This measures simple function calls that are not methods, do not use varargs or kwargs, and do not use tuple unpacking. Taken from: https://github.com/python/performance/blob/9b8d859/performance/benchmarks/bm_call_simple.py """ import weetest def foo(a, b, c, d): # 20 calls bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) bar(a, b, c) def bar(a, b, c): # 20 calls baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) baz(a, b) def baz(a, b): # 20 calls quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) quux(a) def quux(a): # 20 calls qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() qux() def qux(): pass def BenchmarkCallSimple(b): for _ in xrange(b.N): # 20 calls foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) foo(1, 2, 3, 4) if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: benchmarks/call.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Benchmarks for function calls.""" # pylint: disable=unused-argument import weetest def BenchmarkCallNoArgs(b): def Foo(): pass for _ in xrange(b.N): Foo() def BenchmarkCallPositionalArgs(b): def Foo(a, b, c): pass for _ in xrange(b.N): Foo(1, 2, 3) def BenchmarkCallKeywords(b): def Foo(a, b, c): pass for _ in xrange(b.N): Foo(a=1, b=2, c=3) def BenchmarkCallDefaults(b): def Foo(a=1, b=2, c=3): pass for _ in xrange(b.N): Foo() def BenchmarkCallVarArgs(b): def Foo(*args): pass for _ in xrange(b.N): Foo(1, 2, 3) def BenchmarkCallKwargs(b): def Foo(**kwargs): pass for _ in xrange(b.N): Foo(a=1, b=2, c=3) if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: benchmarks/comprehension.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Benchmarks for comprehensions.""" # pylint: disable=unused-argument import weetest def BenchmarkGeneratorExpCreate(b): l = [] for _ in xrange(b.N): (x for x in l) # pylint: disable=pointless-statement def BenchmarkGeneratorExpIterate(b): for _ in (x for x in xrange(b.N)): pass def BenchmarkListCompCreate(b): for _ in xrange(b.N): [x for x in xrange(1000)] # pylint: disable=expression-not-assigned def BenchmarkDictCompCreate(b): for _ in xrange(b.N): {x: x for x in xrange(1000)} # pylint: disable=expression-not-assigned if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: benchmarks/concurrent.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Benchmarks for simple parallel calculations.""" import threading import weetest def Arithmetic(n): return n * 3 + 2 def Fib(n): if n < 2: return 1 return Fib(n - 1) + Fib(n - 2) _WORKLOADS = [ (Arithmetic, 1001), (Fib, 10), ] def _MakeParallelBenchmark(p, work_func, *args): """Create and return a benchmark that runs work_func p times in parallel.""" def Benchmark(b): # pylint: disable=missing-docstring e = threading.Event() def Target(): e.wait() for _ in xrange(b.N / p): work_func(*args) threads = [] for _ in xrange(p): t = threading.Thread(target=Target) t.start() threads.append(t) b.ResetTimer() e.set() for t in threads: t.join() return Benchmark def _RegisterBenchmarks(): for p in xrange(1, 13): for work_func, arg in _WORKLOADS: name = 'Benchmark' + work_func.__name__ if p > 1: name += 'Parallel%s' % p globals()[name] = _MakeParallelBenchmark(p, work_func, arg) _RegisterBenchmarks() if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: benchmarks/dict.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Benchmarks for dictionary operations.""" # pylint: disable=pointless-statement import weetest def BenchmarkDictCreate(b): for _ in xrange(b.N): d = {'one': 1, 'two': 2, 'three': 3} def BenchmarkDictCreateFunc(b): for _ in xrange(b.N): d = dict(one=1, two=2, three=3) def BenchmarkDictGetItem(b): d = {42: 123} for _ in xrange(b.N): d[42] def BenchmarkDictStringOnlyGetItem(b): d = {'foo': 123} for _ in xrange(b.N): d['foo'] def BenchmarkDictSetItem(b): d = {} for _ in xrange(b.N): d[42] = 123 def BenchmarkDictStringOnlySetItem(b): d = {} for _ in xrange(b.N): d['foo'] = 123 def BenchmarkHashStrCached(b): """Hashes the same value repeatedly to exercise any hash caching logic.""" h = hash # Prevent builtins lookup each iteration. for _ in xrange(b.N): h('foobarfoobarfoobar') if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: benchmarks/generator.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Benchmarks for generator functions.""" import weetest def BenchmarkGeneratorIterate(b): def Gen(n): for i in xrange(n): yield i for _ in Gen(b.N): pass def BenchmarkGeneratorCreate(b): def Gen(): yield 1 for _ in xrange(b.N): Gen() if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: benchmarks/list.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Benchmarks for list operations.""" # pylint: disable=pointless-statement import weetest def BenchmarkListGetItem(b): l = [1, 3, 9] for _ in xrange(b.N): l[2] def BenchmarkListContains3(b): l = [1, 3, 9] for _ in xrange(b.N): 9 in l def BenchmarkListContains10(b): l = range(10) for _ in xrange(b.N): 9 in l if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: benchmarks/loop.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Benchmarks for simple low overhead loops.""" import weetest def BenchmarkForXRange(b): for _ in xrange(b.N): pass def BenchmarkWhileCounter(b): i = 0 while i < b.N: i += 1 def BenchmarkWhileXRange(b): i = iter(xrange(b.N)) try: while True: i.next() except StopIteration: pass if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: benchmarks/tuple.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Benchmarks for list operations.""" # pylint: disable=pointless-statement import weetest def BenchmarkTupleGetItem(b): l = (1, 3, 9) for _ in xrange(b.N): l[2] def BenchmarkTupleContains3(b): t = (1, 3, 9) for _ in xrange(b.N): 9 in t def BenchmarkTupleContains10(b): t = tuple(range(10)) for _ in xrange(b.N): 9 in t if __name__ == '__main__': weetest.RunBenchmarks() ================================================ FILE: compiler/__init__.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ================================================ FILE: compiler/block.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Classes for analyzing and storing the state of Python code blocks.""" from __future__ import unicode_literals import abc import collections import re from grumpy.compiler import expr from grumpy.compiler import util from grumpy.pythonparser import algorithm from grumpy.pythonparser import ast from grumpy.pythonparser import source _non_word_re = re.compile('[^A-Za-z0-9_]') class Package(object): """A Go package import.""" def __init__(self, name, alias=None): self.name = name # Use Γ as a separator since it provides readability with a low # probability of name collisions. self.alias = alias or 'π_' + name.replace('/', 'Γ').replace('.', 'Γ') class Loop(object): """Represents a for or while loop within a particular block.""" def __init__(self, breakvar): self.breakvar = breakvar class Block(object): """Represents a Python block such as a function or class definition.""" __metaclass__ = abc.ABCMeta def __init__(self, parent, name): self.root = parent.root if parent else self self.parent = parent self.name = name self.free_temps = set() self.used_temps = set() self.temp_index = 0 self.label_count = 0 self.checkpoints = set() self.loop_stack = [] self.is_generator = False @abc.abstractmethod def bind_var(self, writer, name, value): """Writes Go statements for assigning value to named var in this block. This is overridden in the different concrete block types since in Python, binding a variable in, e.g. a function is quite different than binding at global block. Args: writer: The Writer object where statements will be written. name: The name of the Python variable. value: A Go expression to assign to the variable. """ pass @abc.abstractmethod def del_var(self, writer, name): pass @abc.abstractmethod def resolve_name(self, writer, name): """Returns a GeneratedExpr object for accessing the named var in this block. This is overridden in the different concrete block types since name resolution in Python behaves differently depending on where in what kind of block its happening within, e.g. local vars are different than globals. Args: writer: Writer object where intermediate calculations will be printed. name: The name of the Python variable. """ pass def genlabel(self, is_checkpoint=False): self.label_count += 1 if is_checkpoint: self.checkpoints.add(self.label_count) return self.label_count def alloc_temp(self, type_='*πg.Object'): """Create a new temporary Go variable having type type_ for this block.""" for v in sorted(self.free_temps, key=lambda k: k.name): if v.type_ == type_: self.free_temps.remove(v) self.used_temps.add(v) return v self.temp_index += 1 name = 'πTemp{:03d}'.format(self.temp_index) v = expr.GeneratedTempVar(self, name, type_) self.used_temps.add(v) return v def free_temp(self, v): """Release the GeneratedTempVar v so it can be reused.""" self.used_temps.remove(v) self.free_temps.add(v) def push_loop(self, breakvar): loop = Loop(breakvar) self.loop_stack.append(loop) return loop def pop_loop(self): self.loop_stack.pop() def top_loop(self): return self.loop_stack[-1] def _resolve_global(self, writer, name): result = self.alloc_temp() writer.write_checked_call2( result, 'πg.ResolveGlobal(πF, {})', self.root.intern(name)) return result class ModuleBlock(Block): """Python block for a module.""" def __init__(self, importer, full_package_name, filename, src, future_features): Block.__init__(self, None, '') self.importer = importer self.full_package_name = full_package_name self.filename = filename self.buffer = source.Buffer(src) self.strings = set() self.future_features = future_features def bind_var(self, writer, name, value): writer.write_checked_call1( 'πF.Globals().SetItem(πF, {}.ToObject(), {})', self.intern(name), value) def del_var(self, writer, name): writer.write_checked_call1('πg.DelVar(πF, πF.Globals(), {})', self.intern(name)) def resolve_name(self, writer, name): return self._resolve_global(writer, name) def intern(self, s): if len(s) > 64 or _non_word_re.search(s): return 'πg.NewStr({})'.format(util.go_str(s)) self.strings.add(s) return 'ß' + s class ClassBlock(Block): """Python block for a class definition.""" def __init__(self, parent, name, global_vars): Block.__init__(self, parent, name) self.global_vars = global_vars def bind_var(self, writer, name, value): if name in self.global_vars: return self.root.bind_var(writer, name, value) writer.write_checked_call1('πClass.SetItem(πF, {}.ToObject(), {})', self.root.intern(name), value) def del_var(self, writer, name): if name in self.global_vars: return self.root.del_var(writer, name) writer.write_checked_call1('πg.DelVar(πF, πClass, {})', self.root.intern(name)) def resolve_name(self, writer, name): local = 'nil' if name not in self.global_vars: # Only look for a local in an outer block when name hasn't been declared # global in this block. If it has been declared global then we fallback # straight to the global dict. block = self.parent while not isinstance(block, ModuleBlock): if isinstance(block, FunctionBlock) and name in block.vars: var = block.vars[name] if var.type != Var.TYPE_GLOBAL: local = util.adjust_local_name(name) # When it is declared global, prefer it to anything in outer blocks. break block = block.parent result = self.alloc_temp() writer.write_checked_call2( result, 'πg.ResolveClass(πF, πClass, {}, {})', local, self.root.intern(name)) return result class FunctionBlock(Block): """Python block for a function definition.""" def __init__(self, parent, name, block_vars, is_generator): Block.__init__(self, parent, name) self.vars = block_vars self.parent = parent self.is_generator = is_generator def bind_var(self, writer, name, value): if self.vars[name].type == Var.TYPE_GLOBAL: return self.root.bind_var(writer, name, value) writer.write('{} = {}'.format(util.adjust_local_name(name), value)) def del_var(self, writer, name): var = self.vars.get(name) if not var: raise util.ParseError( None, 'cannot delete nonexistent local: {}'.format(name)) if var.type == Var.TYPE_GLOBAL: return self.root.del_var(writer, name) adjusted_name = util.adjust_local_name(name) # Resolve local first to ensure the variable is already bound. writer.write_checked_call1('πg.CheckLocal(πF, {}, {})', adjusted_name, util.go_str(name)) writer.write('{} = πg.UnboundLocal'.format(adjusted_name)) def resolve_name(self, writer, name): block = self while not isinstance(block, ModuleBlock): if isinstance(block, FunctionBlock): var = block.vars.get(name) if var: if var.type == Var.TYPE_GLOBAL: return self._resolve_global(writer, name) writer.write_checked_call1('πg.CheckLocal(πF, {}, {})', util.adjust_local_name(name), util.go_str(name)) return expr.GeneratedLocalVar(name) block = block.parent return self._resolve_global(writer, name) class Var(object): """A Python variable used within a particular block.""" TYPE_LOCAL = 0 TYPE_PARAM = 1 TYPE_GLOBAL = 2 def __init__(self, name, var_type, arg_index=None): self.name = name self.type = var_type if var_type == Var.TYPE_LOCAL: assert arg_index is None self.init_expr = 'πg.UnboundLocal' elif var_type == Var.TYPE_PARAM: assert arg_index is not None self.init_expr = 'πArgs[{}]'.format(arg_index) else: assert arg_index is None self.init_expr = None class BlockVisitor(algorithm.Visitor): """Visits nodes in a function or class to determine block variables.""" # pylint: disable=invalid-name,missing-docstring def __init__(self): self.vars = collections.OrderedDict() def visit_Assign(self, node): for target in node.targets: self._assign_target(target) self.visit(node.value) def visit_AugAssign(self, node): self._assign_target(node.target) self.visit(node.value) def visit_ClassDef(self, node): self._register_local(node.name) def visit_ExceptHandler(self, node): if node.name: self._register_local(node.name.id) self.generic_visit(node) def visit_For(self, node): self._assign_target(node.target) self.generic_visit(node) def visit_FunctionDef(self, node): # The function being defined is local to this block, i.e. is nested within # another function. Note that further nested symbols are not traversed # because we don't explicitly visit the function body. self._register_local(node.name) def visit_Global(self, node): for name in node.names: self._register_global(node, name) def visit_Import(self, node): for alias in node.names: self._register_local(alias.asname or alias.name.split('.')[0]) def visit_ImportFrom(self, node): for alias in node.names: self._register_local(alias.asname or alias.name) def visit_With(self, node): for item in node.items: if item.optional_vars: self._assign_target(item.optional_vars) self.generic_visit(node) def _assign_target(self, target): if isinstance(target, ast.Name): self._register_local(target.id) elif isinstance(target, (ast.Tuple, ast.List)): for elt in target.elts: self._assign_target(elt) def _register_global(self, node, name): var = self.vars.get(name) if var: if var.type == Var.TYPE_PARAM: msg = "name '{}' is parameter and global" raise util.ParseError(node, msg.format(name)) if var.type == Var.TYPE_LOCAL: msg = "name '{}' is used prior to global declaration" raise util.ParseError(node, msg.format(name)) else: self.vars[name] = Var(name, Var.TYPE_GLOBAL) def _register_local(self, name): if not self.vars.get(name): self.vars[name] = Var(name, Var.TYPE_LOCAL) class FunctionBlockVisitor(BlockVisitor): """Visits function nodes to determine variables and generator state.""" # pylint: disable=invalid-name,missing-docstring def __init__(self, node): BlockVisitor.__init__(self) self.is_generator = False node_args = node.args args = [a.arg for a in node_args.args] if node_args.vararg: args.append(node_args.vararg.arg) if node_args.kwarg: args.append(node_args.kwarg.arg) for i, name in enumerate(args): if name in self.vars: msg = "duplicate argument '{}' in function definition".format(name) raise util.ParseError(node, msg) self.vars[name] = Var(name, Var.TYPE_PARAM, arg_index=i) def visit_Yield(self, unused_node): # pylint: disable=unused-argument self.is_generator = True ================================================ FILE: compiler/block_test.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests Package, Block, BlockVisitor and related classes.""" from __future__ import unicode_literals import textwrap import unittest from grumpy.compiler import block from grumpy.compiler import imputil from grumpy.compiler import util from grumpy import pythonparser class PackageTest(unittest.TestCase): def testCreate(self): package = block.Package('foo/bar/baz') self.assertEqual(package.name, 'foo/bar/baz') self.assertEqual(package.alias, 'π_fooΓbarΓbaz') def testCreateGrump(self): package = block.Package('foo/bar/baz', 'myalias') self.assertEqual(package.name, 'foo/bar/baz') self.assertEqual(package.alias, 'myalias') class BlockTest(unittest.TestCase): def testLoop(self): b = _MakeModuleBlock() loop = b.push_loop(None) self.assertEqual(loop, b.top_loop()) inner_loop = b.push_loop(None) self.assertEqual(inner_loop, b.top_loop()) b.pop_loop() self.assertEqual(loop, b.top_loop()) def testResolveName(self): module_block = _MakeModuleBlock() block_vars = {'foo': block.Var('foo', block.Var.TYPE_LOCAL)} func1_block = block.FunctionBlock(module_block, 'func1', block_vars, False) block_vars = {'bar': block.Var('bar', block.Var.TYPE_LOCAL)} func2_block = block.FunctionBlock(func1_block, 'func2', block_vars, False) block_vars = {'case': block.Var('case', block.Var.TYPE_LOCAL)} keyword_block = block.FunctionBlock( module_block, 'keyword_func', block_vars, False) class1_block = block.ClassBlock(module_block, 'Class1', set()) class2_block = block.ClassBlock(func1_block, 'Class2', set()) self.assertRegexpMatches(self._ResolveName(module_block, 'foo'), r'ResolveGlobal\b.*foo') self.assertRegexpMatches(self._ResolveName(module_block, 'bar'), r'ResolveGlobal\b.*bar') self.assertRegexpMatches(self._ResolveName(module_block, 'baz'), r'ResolveGlobal\b.*baz') self.assertRegexpMatches(self._ResolveName(func1_block, 'foo'), r'CheckLocal\b.*foo') self.assertRegexpMatches(self._ResolveName(func1_block, 'bar'), r'ResolveGlobal\b.*bar') self.assertRegexpMatches(self._ResolveName(func1_block, 'baz'), r'ResolveGlobal\b.*baz') self.assertRegexpMatches(self._ResolveName(func2_block, 'foo'), r'CheckLocal\b.*foo') self.assertRegexpMatches(self._ResolveName(func2_block, 'bar'), r'CheckLocal\b.*bar') self.assertRegexpMatches(self._ResolveName(func2_block, 'baz'), r'ResolveGlobal\b.*baz') self.assertRegexpMatches(self._ResolveName(class1_block, 'foo'), r'ResolveClass\(.*, nil, .*foo') self.assertRegexpMatches(self._ResolveName(class2_block, 'foo'), r'ResolveClass\(.*, µfoo, .*foo') self.assertRegexpMatches(self._ResolveName(keyword_block, 'case'), r'CheckLocal\b.*µcase, "case"') def _ResolveName(self, b, name): writer = util.Writer() b.resolve_name(writer, name) return writer.getvalue() class BlockVisitorTest(unittest.TestCase): def testAssignSingle(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('foo = 3')) self.assertEqual(visitor.vars.keys(), ['foo']) self.assertRegexpMatches(visitor.vars['foo'].init_expr, r'UnboundLocal') def testAssignMultiple(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('foo = bar = 123')) self.assertEqual(sorted(visitor.vars.keys()), ['bar', 'foo']) self.assertRegexpMatches(visitor.vars['foo'].init_expr, r'UnboundLocal') self.assertRegexpMatches(visitor.vars['bar'].init_expr, r'UnboundLocal') def testAssignTuple(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('foo, bar = "a", "b"')) self.assertEqual(sorted(visitor.vars.keys()), ['bar', 'foo']) self.assertRegexpMatches(visitor.vars['foo'].init_expr, r'UnboundLocal') self.assertRegexpMatches(visitor.vars['bar'].init_expr, r'UnboundLocal') def testAssignNested(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('foo, (bar, baz) = "a", ("b", "c")')) self.assertEqual(sorted(visitor.vars.keys()), ['bar', 'baz', 'foo']) self.assertRegexpMatches(visitor.vars['foo'].init_expr, r'UnboundLocal') self.assertRegexpMatches(visitor.vars['bar'].init_expr, r'UnboundLocal') self.assertRegexpMatches(visitor.vars['baz'].init_expr, r'UnboundLocal') def testAugAssignSingle(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('foo += 3')) self.assertEqual(visitor.vars.keys(), ['foo']) self.assertRegexpMatches(visitor.vars['foo'].init_expr, r'UnboundLocal') def testVisitClassDef(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('class Foo(object): pass')) self.assertEqual(visitor.vars.keys(), ['Foo']) self.assertRegexpMatches(visitor.vars['Foo'].init_expr, r'UnboundLocal') def testExceptHandler(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt(textwrap.dedent("""\ try: pass except Exception as foo: pass except TypeError as bar: pass"""))) self.assertEqual(sorted(visitor.vars.keys()), ['bar', 'foo']) self.assertRegexpMatches(visitor.vars['foo'].init_expr, r'UnboundLocal') self.assertRegexpMatches(visitor.vars['bar'].init_expr, r'UnboundLocal') def testFor(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('for i in foo: pass')) self.assertEqual(visitor.vars.keys(), ['i']) self.assertRegexpMatches(visitor.vars['i'].init_expr, r'UnboundLocal') def testFunctionDef(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('def foo(): pass')) self.assertEqual(visitor.vars.keys(), ['foo']) self.assertRegexpMatches(visitor.vars['foo'].init_expr, r'UnboundLocal') def testImport(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('import foo.bar, baz')) self.assertEqual(sorted(visitor.vars.keys()), ['baz', 'foo']) self.assertRegexpMatches(visitor.vars['foo'].init_expr, r'UnboundLocal') self.assertRegexpMatches(visitor.vars['baz'].init_expr, r'UnboundLocal') def testImportFrom(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('from foo.bar import baz, qux')) self.assertEqual(sorted(visitor.vars.keys()), ['baz', 'qux']) self.assertRegexpMatches(visitor.vars['baz'].init_expr, r'UnboundLocal') self.assertRegexpMatches(visitor.vars['qux'].init_expr, r'UnboundLocal') def testGlobal(self): visitor = block.BlockVisitor() visitor.visit(_ParseStmt('global foo, bar')) self.assertEqual(sorted(visitor.vars.keys()), ['bar', 'foo']) self.assertIsNone(visitor.vars['foo'].init_expr) self.assertIsNone(visitor.vars['bar'].init_expr) def testGlobalIsParam(self): visitor = block.BlockVisitor() visitor.vars['foo'] = block.Var('foo', block.Var.TYPE_PARAM, arg_index=0) self.assertRaisesRegexp(util.ParseError, 'is parameter and global', visitor.visit, _ParseStmt('global foo')) def testGlobalUsedPriorToDeclaration(self): node = pythonparser.parse('foo = 42\nglobal foo') visitor = block.BlockVisitor() self.assertRaisesRegexp(util.ParseError, 'used prior to global declaration', visitor.generic_visit, node) class FunctionBlockVisitorTest(unittest.TestCase): def testArgs(self): func = _ParseStmt('def foo(bar, baz, *args, **kwargs): pass') visitor = block.FunctionBlockVisitor(func) self.assertIn('bar', visitor.vars) self.assertIn('baz', visitor.vars) self.assertIn('args', visitor.vars) self.assertIn('kwargs', visitor.vars) self.assertRegexpMatches(visitor.vars['bar'].init_expr, r'Args\[0\]') self.assertRegexpMatches(visitor.vars['baz'].init_expr, r'Args\[1\]') self.assertRegexpMatches(visitor.vars['args'].init_expr, r'Args\[2\]') self.assertRegexpMatches(visitor.vars['kwargs'].init_expr, r'Args\[3\]') def testArgsDuplicate(self): func = _ParseStmt('def foo(bar, baz, bar=None): pass') self.assertRaisesRegexp(util.ParseError, 'duplicate argument', block.FunctionBlockVisitor, func) def testYield(self): visitor = block.FunctionBlockVisitor(_ParseStmt('def foo(): pass')) visitor.visit(_ParseStmt('yield "foo"')) self.assertTrue(visitor.is_generator) def testYieldExpr(self): visitor = block.FunctionBlockVisitor(_ParseStmt('def foo(): pass')) visitor.visit(_ParseStmt('foo = (yield)')) self.assertTrue(visitor.is_generator) self.assertEqual(sorted(visitor.vars.keys()), ['foo']) self.assertRegexpMatches(visitor.vars['foo'].init_expr, r'UnboundLocal') def _MakeModuleBlock(): importer = imputil.Importer(None, '__main__', '/tmp/foo.py', False) return block.ModuleBlock(importer, '__main__', '', '', imputil.FutureFeatures()) def _ParseStmt(stmt_str): return pythonparser.parse(stmt_str).body[0] if __name__ == '__main__': unittest.main() ================================================ FILE: compiler/expr.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Classes representing generated expressions.""" from __future__ import unicode_literals import abc from grumpy.compiler import util class GeneratedExpr(object): """GeneratedExpr is a generated Go expression in transcompiled output.""" __metaclass__ = abc.ABCMeta def __enter__(self): return self def __exit__(self, unused_type, unused_value, unused_traceback): self.free() @abc.abstractproperty def expr(self): pass def free(self): pass class GeneratedTempVar(GeneratedExpr): """GeneratedTempVar is an expression result stored in a temporary value.""" def __init__(self, block_, name, type_): self.block = block_ self.name = name self.type_ = type_ @property def expr(self): return self.name def free(self): self.block.free_temp(self) class GeneratedLocalVar(GeneratedExpr): """GeneratedLocalVar is the Go local var corresponding to a Python local.""" def __init__(self, name): self._name = name @property def expr(self): return util.adjust_local_name(self._name) class GeneratedLiteral(GeneratedExpr): """GeneratedLiteral is a generated literal Go expression.""" def __init__(self, expr): self._expr = expr @property def expr(self): return self._expr nil_expr = GeneratedLiteral('nil') class BlankVar(GeneratedExpr): def __init__(self): self.name = '_' @property def expr(self): return '_' blank_var = BlankVar() ================================================ FILE: compiler/expr_visitor.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Visitor class for traversing Python expressions.""" from __future__ import unicode_literals import contextlib import textwrap from grumpy.compiler import expr from grumpy.compiler import util from grumpy.pythonparser import algorithm from grumpy.pythonparser import ast class ExprVisitor(algorithm.Visitor): """Builds and returns a Go expression representing the Python nodes.""" # pylint: disable=invalid-name,missing-docstring def __init__(self, stmt_visitor): self.stmt_visitor = stmt_visitor self.block = stmt_visitor.block self.writer = stmt_visitor.writer def generic_visit(self, node): msg = 'expression node not yet implemented: ' + type(node).__name__ raise util.ParseError(node, msg) def visit_Attribute(self, node): with self.visit(node.value) as obj: attr = self.block.alloc_temp() self.writer.write_checked_call2( attr, 'πg.GetAttr(πF, {}, {}, nil)', obj.expr, self.block.root.intern(node.attr)) return attr def visit_BinOp(self, node): result = self.block.alloc_temp() with self.visit(node.left) as lhs, self.visit(node.right) as rhs: op_type = type(node.op) if op_type in ExprVisitor._BIN_OP_TEMPLATES: tmpl = ExprVisitor._BIN_OP_TEMPLATES[op_type] self.writer.write_checked_call2( result, tmpl, lhs=lhs.expr, rhs=rhs.expr) else: msg = 'binary op not implemented: {}'.format(op_type.__name__) raise util.ParseError(node, msg) return result def visit_BoolOp(self, node): result = self.block.alloc_temp() with self.block.alloc_temp('bool') as is_true: if isinstance(node.op, ast.And): cond_expr = '!' + is_true.expr else: cond_expr = is_true.expr end_label = self.block.genlabel() num_values = len(node.values) for i, n in enumerate(node.values): with self.visit(n) as v: self.writer.write('{} = {}'.format(result.expr, v.expr)) if i < num_values - 1: self.writer.write_checked_call2( is_true, 'πg.IsTrue(πF, {})', result.expr) self.writer.write_tmpl(textwrap.dedent("""\ if $cond_expr { \tgoto Label$end_label }"""), cond_expr=cond_expr, end_label=end_label) self.writer.write_label(end_label) return result def visit_Call(self, node): # Build positional arguments. args = expr.nil_expr if node.args: args = self.block.alloc_temp('[]*πg.Object') self.writer.write('{} = πF.MakeArgs({})'.format(args.expr, len(node.args))) for i, n in enumerate(node.args): with self.visit(n) as a: self.writer.write('{}[{}] = {}'.format(args.expr, i, a.expr)) varg = expr.nil_expr if node.starargs: varg = self.visit(node.starargs) # Build keyword arguments keywords = expr.nil_expr if node.keywords: values = [] for k in node.keywords: values.append((util.go_str(k.arg), self.visit(k.value))) keywords = self.block.alloc_temp('πg.KWArgs') self.writer.write_tmpl('$keywords = πg.KWArgs{', keywords=keywords.name) with self.writer.indent_block(): for k, v in values: with v: self.writer.write_tmpl('{$name, $value},', name=k, value=v.expr) self.writer.write('}') kwargs = expr.nil_expr if node.kwargs: kwargs = self.visit(node.kwargs) # Invoke function with all parameters. with args, varg, keywords, kwargs, self.visit(node.func) as func: result = self.block.alloc_temp() if varg is expr.nil_expr and kwargs is expr.nil_expr: self.writer.write_checked_call2(result, '{}.Call(πF, {}, {})', func.expr, args.expr, keywords.expr) else: self.writer.write_checked_call2(result, 'πg.Invoke(πF, {}, {}, {}, {}, {})', func.expr, args.expr, varg.expr, keywords.expr, kwargs.expr) if node.args: self.writer.write('πF.FreeArgs({})'.format(args.expr)) return result def visit_Compare(self, node): result = self.block.alloc_temp() lhs = self.visit(node.left) n = len(node.ops) end_label = self.block.genlabel() if n > 1 else None for i, (op, comp) in enumerate(zip(node.ops, node.comparators)): rhs = self.visit(comp) op_type = type(op) if op_type in ExprVisitor._CMP_OP_TEMPLATES: tmpl = ExprVisitor._CMP_OP_TEMPLATES[op_type] self.writer.write_checked_call2( result, tmpl, lhs=lhs.expr, rhs=rhs.expr) elif isinstance(op, (ast.In, ast.NotIn)): with self.block.alloc_temp('bool') as contains: self.writer.write_checked_call2( contains, 'πg.Contains(πF, {}, {})', rhs.expr, lhs.expr) invert = '' if isinstance(op, ast.In) else '!' self.writer.write('{} = πg.GetBool({}{}).ToObject()'.format( result.name, invert, contains.expr)) elif isinstance(op, ast.Is): self.writer.write('{} = πg.GetBool({} == {}).ToObject()'.format( result.name, lhs.expr, rhs.expr)) elif isinstance(op, ast.IsNot): self.writer.write('{} = πg.GetBool({} != {}).ToObject()'.format( result.name, lhs.expr, rhs.expr)) else: raise AssertionError('unrecognized compare op: {}'.format( op_type.__name__)) if i < n - 1: with self.block.alloc_temp('bool') as cond: self.writer.write_checked_call2( cond, 'πg.IsTrue(πF, {})', result.expr) self.writer.write_tmpl(textwrap.dedent("""\ if !$cond { \tgoto Label$end_label }"""), cond=cond.expr, end_label=end_label) lhs.free() lhs = rhs rhs.free() if end_label is not None: self.writer.write_label(end_label) return result def visit_Dict(self, node): with self.block.alloc_temp('*πg.Dict') as d: self.writer.write('{} = πg.NewDict()'.format(d.name)) for k, v in zip(node.keys, node.values): with self.visit(k) as key, self.visit(v) as value: self.writer.write_checked_call1('{}.SetItem(πF, {}, {})', d.expr, key.expr, value.expr) result = self.block.alloc_temp() self.writer.write('{} = {}.ToObject()'.format(result.name, d.expr)) return result def visit_Set(self, node): with self.block.alloc_temp('*πg.Set') as s: self.writer.write('{} = πg.NewSet()'.format(s.name)) for e in node.elts: with self.visit(e) as value: self.writer.write_checked_call2(expr.blank_var, '{}.Add(πF, {})', s.expr, value.expr) result = self.block.alloc_temp() self.writer.write('{} = {}.ToObject()'.format(result.name, s.expr)) return result def visit_DictComp(self, node): result = self.block.alloc_temp() elt = ast.Tuple(elts=[node.key, node.value]) gen_node = ast.GeneratorExp( elt=elt, generators=node.generators, loc=node.loc) with self.visit(gen_node) as gen: self.writer.write_checked_call2( result, 'πg.DictType.Call(πF, πg.Args{{{}}}, nil)', gen.expr) return result def visit_ExtSlice(self, node): result = self.block.alloc_temp() if len(node.dims) <= util.MAX_DIRECT_TUPLE: with contextlib.nested(*(self.visit(d) for d in node.dims)) as dims: self.writer.write('{} = πg.NewTuple{}({}).ToObject()'.format( result.name, len(dims), ', '.join(d.expr for d in dims))) else: with self.block.alloc_temp('[]*πg.Object') as dims: self.writer.write('{} = make([]*πg.Object, {})'.format( dims.name, len(node.dims))) for i, dim in enumerate(node.dims): with self.visit(dim) as s: self.writer.write('{}[{}] = {}'.format(dims.name, i, s.expr)) self.writer.write('{} = πg.NewTuple({}...).ToObject()'.format( result.name, dims.expr)) return result def visit_GeneratorExp(self, node): body = ast.Expr(value=ast.Yield(value=node.elt), loc=node.loc) for comp_node in reversed(node.generators): for if_node in reversed(comp_node.ifs): body = ast.If(test=if_node, body=[body], orelse=[], loc=node.loc) # pylint: disable=redefined-variable-type body = ast.For(target=comp_node.target, iter=comp_node.iter, body=[body], orelse=[], loc=node.loc) args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]) node = ast.FunctionDef(name='', args=args, body=[body]) gen_func = self.stmt_visitor.visit_function_inline(node) result = self.block.alloc_temp() self.writer.write_checked_call2( result, '{}.Call(πF, nil, nil)', gen_func.expr) return result def visit_IfExp(self, node): else_label, end_label = self.block.genlabel(), self.block.genlabel() result = self.block.alloc_temp() with self.visit(node.test) as test, self.block.alloc_temp('bool') as cond: self.writer.write_checked_call2( cond, 'πg.IsTrue(πF, {})', test.expr) self.writer.write_tmpl(textwrap.dedent("""\ if !$cond { \tgoto Label$else_label }"""), cond=cond.expr, else_label=else_label) with self.visit(node.body) as value: self.writer.write('{} = {}'.format(result.name, value.expr)) self.writer.write('goto Label{}'.format(end_label)) self.writer.write_label(else_label) with self.visit(node.orelse) as value: self.writer.write('{} = {}'.format(result.name, value.expr)) self.writer.write_label(end_label) return result def visit_Index(self, node): result = self.block.alloc_temp() with self.visit(node.value) as v: self.writer.write('{} = {}'.format(result.name, v.expr)) return result def visit_Lambda(self, node): ret = ast.Return(value=node.body, loc=node.loc) func_node = ast.FunctionDef( name='', args=node.args, body=[ret]) return self.stmt_visitor.visit_function_inline(func_node) def visit_List(self, node): with self._visit_seq_elts(node.elts) as elems: result = self.block.alloc_temp() self.writer.write('{} = πg.NewList({}...).ToObject()'.format( result.expr, elems.expr)) return result def visit_ListComp(self, node): result = self.block.alloc_temp() gen_node = ast.GeneratorExp( elt=node.elt, generators=node.generators, loc=node.loc) with self.visit(gen_node) as gen: self.writer.write_checked_call2( result, 'πg.ListType.Call(πF, πg.Args{{{}}}, nil)', gen.expr) return result def visit_Name(self, node): return self.block.resolve_name(self.writer, node.id) def visit_Num(self, node): if isinstance(node.n, int): expr_str = 'NewInt({})'.format(node.n) elif isinstance(node.n, long): a = abs(node.n) gobytes = '' while a: gobytes = hex(int(a&255)) + ',' + gobytes a >>= 8 expr_str = 'NewLongFromBytes([]byte{{{}}})'.format(gobytes) if node.n < 0: expr_str = expr_str + '.Neg()' elif isinstance(node.n, float): expr_str = 'NewFloat({})'.format(node.n) elif isinstance(node.n, complex): expr_str = 'NewComplex(complex({}, {}))'.format(node.n.real, node.n.imag) else: msg = 'number type not yet implemented: ' + type(node.n).__name__ raise util.ParseError(node, msg) return expr.GeneratedLiteral('πg.' + expr_str + '.ToObject()') def visit_Slice(self, node): result = self.block.alloc_temp() lower = upper = step = expr.GeneratedLiteral('πg.None') if node.lower: lower = self.visit(node.lower) if node.upper: upper = self.visit(node.upper) if node.step: step = self.visit(node.step) with lower, upper, step: self.writer.write_checked_call2( result, 'πg.SliceType.Call(πF, πg.Args{{{}, {}, {}}}, nil)', lower.expr, upper.expr, step.expr) return result def visit_Subscript(self, node): rhs = self.visit(node.slice) result = self.block.alloc_temp() with rhs, self.visit(node.value) as lhs: self.writer.write_checked_call2(result, 'πg.GetItem(πF, {}, {})', lhs.expr, rhs.expr) return result def visit_Str(self, node): if isinstance(node.s, unicode): expr_str = 'πg.NewUnicode({}).ToObject()'.format( util.go_str(node.s.encode('utf-8'))) else: expr_str = '{}.ToObject()'.format(self.block.root.intern(node.s)) return expr.GeneratedLiteral(expr_str) def visit_Tuple(self, node): result = self.block.alloc_temp() if len(node.elts) <= util.MAX_DIRECT_TUPLE: with contextlib.nested(*(self.visit(e) for e in node.elts)) as elts: self.writer.write('{} = πg.NewTuple{}({}).ToObject()'.format( result.name, len(elts), ', '.join(e.expr for e in elts))) else: with self._visit_seq_elts(node.elts) as elems: self.writer.write('{} = πg.NewTuple({}...).ToObject()'.format( result.expr, elems.expr)) return result def visit_UnaryOp(self, node): result = self.block.alloc_temp() with self.visit(node.operand) as operand: op_type = type(node.op) if op_type in ExprVisitor._UNARY_OP_TEMPLATES: self.writer.write_checked_call2( result, ExprVisitor._UNARY_OP_TEMPLATES[op_type], operand=operand.expr) elif isinstance(node.op, ast.Not): with self.block.alloc_temp('bool') as is_true: self.writer.write_checked_call2( is_true, 'πg.IsTrue(πF, {})', operand.expr) self.writer.write('{} = πg.GetBool(!{}).ToObject()'.format( result.name, is_true.expr)) else: msg = 'unary op not implemented: {}'.format(op_type.__name__) raise util.ParseError(node, msg) return result def visit_Yield(self, node): if node.value: value = self.visit(node.value) else: value = expr.GeneratedLiteral('πg.None') resume_label = self.block.genlabel(is_checkpoint=True) self.writer.write('πF.PushCheckpoint({})'.format(resume_label)) self.writer.write('return {}, nil'.format(value.expr)) self.writer.write_label(resume_label) result = self.block.alloc_temp() self.writer.write('{} = πSent'.format(result.name)) return result _BIN_OP_TEMPLATES = { ast.BitAnd: 'πg.And(πF, {lhs}, {rhs})', ast.BitOr: 'πg.Or(πF, {lhs}, {rhs})', ast.BitXor: 'πg.Xor(πF, {lhs}, {rhs})', ast.Add: 'πg.Add(πF, {lhs}, {rhs})', ast.Div: 'πg.Div(πF, {lhs}, {rhs})', # TODO: Support "from __future__ import division". ast.FloorDiv: 'πg.FloorDiv(πF, {lhs}, {rhs})', ast.LShift: 'πg.LShift(πF, {lhs}, {rhs})', ast.Mod: 'πg.Mod(πF, {lhs}, {rhs})', ast.Mult: 'πg.Mul(πF, {lhs}, {rhs})', ast.Pow: 'πg.Pow(πF, {lhs}, {rhs})', ast.RShift: 'πg.RShift(πF, {lhs}, {rhs})', ast.Sub: 'πg.Sub(πF, {lhs}, {rhs})', } _CMP_OP_TEMPLATES = { ast.Eq: 'πg.Eq(πF, {lhs}, {rhs})', ast.Gt: 'πg.GT(πF, {lhs}, {rhs})', ast.GtE: 'πg.GE(πF, {lhs}, {rhs})', ast.Lt: 'πg.LT(πF, {lhs}, {rhs})', ast.LtE: 'πg.LE(πF, {lhs}, {rhs})', ast.NotEq: 'πg.NE(πF, {lhs}, {rhs})', } _UNARY_OP_TEMPLATES = { ast.Invert: 'πg.Invert(πF, {operand})', ast.UAdd: 'πg.Pos(πF, {operand})', ast.USub: 'πg.Neg(πF, {operand})', } def _visit_seq_elts(self, elts): result = self.block.alloc_temp('[]*πg.Object') self.writer.write('{} = make([]*πg.Object, {})'.format( result.expr, len(elts))) for i, e in enumerate(elts): with self.visit(e) as elt: self.writer.write('{}[{}] = {}'.format(result.expr, i, elt.expr)) return result def _node_not_implemented(self, node): msg = 'node not yet implemented: ' + type(node).__name__ raise util.ParseError(node, msg) visit_SetComp = _node_not_implemented ================================================ FILE: compiler/expr_visitor_test.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for ExprVisitor.""" from __future__ import unicode_literals import subprocess import textwrap import unittest from grumpy.compiler import block from grumpy.compiler import imputil from grumpy.compiler import shard_test from grumpy.compiler import stmt from grumpy import pythonparser def _MakeExprTest(expr): def Test(self): code = 'assert ({}) == ({!r}), {!r}'.format(expr, eval(expr), expr) # pylint: disable=eval-used self.assertEqual((0, ''), _GrumpRun(code)) return Test def _MakeLiteralTest(lit, expected=None): if expected is None: expected = lit def Test(self): status, output = _GrumpRun('print repr({}),'.format(lit)) self.assertEqual(0, status, output) self.assertEqual(expected, output.strip()) # pylint: disable=eval-used return Test def _MakeSliceTest(subscript, want): """Define a test function that evaluates a slice expression.""" def Test(self): code = textwrap.dedent("""\ class Slicer(object): def __getitem__(self, slice): print slice Slicer()[{}]""") status, output = _GrumpRun(code.format(subscript)) self.assertEqual(0, status, output) self.assertEqual(want, output.strip()) return Test class ExprVisitorTest(unittest.TestCase): # pylint: disable=invalid-name def testAttribute(self): code = textwrap.dedent("""\ class Foo(object): bar = 42 assert Foo.bar == 42""") self.assertEqual((0, ''), _GrumpRun(code)) testBinOpArithmeticAdd = _MakeExprTest('1 + 2') testBinOpArithmeticAnd = _MakeExprTest('7 & 12') testBinOpArithmeticDiv = _MakeExprTest('8 / 4') testBinOpArithmeticFloorDiv = _MakeExprTest('8 // 4') testBinOpArithmeticFloorDivRemainder = _MakeExprTest('5 // 2') testBinOpArithmeticMod = _MakeExprTest('9 % 5') testBinOpArithmeticMul = _MakeExprTest('3 * 2') testBinOpArithmeticOr = _MakeExprTest('2 | 6') testBinOpArithmeticPow = _MakeExprTest('2 ** 16') testBinOpArithmeticSub = _MakeExprTest('10 - 3') testBinOpArithmeticXor = _MakeExprTest('3 ^ 5') testBoolOpTrueAndFalse = _MakeExprTest('True and False') testBoolOpTrueAndTrue = _MakeExprTest('True and True') testBoolOpTrueAndExpr = _MakeExprTest('True and 2 == 2') testBoolOpTrueOrFalse = _MakeExprTest('True or False') testBoolOpFalseOrFalse = _MakeExprTest('False or False') testBoolOpFalseOrExpr = _MakeExprTest('False or 2 == 2') def testCall(self): code = textwrap.dedent("""\ def foo(): print 'bar' foo()""") self.assertEqual((0, 'bar\n'), _GrumpRun(code)) def testCallKeywords(self): code = textwrap.dedent("""\ def foo(a=1, b=2): print a, b foo(b=3)""") self.assertEqual((0, '1 3\n'), _GrumpRun(code)) def testCallVarArgs(self): code = textwrap.dedent("""\ def foo(a, b): print a, b foo(*(123, 'abc'))""") self.assertEqual((0, '123 abc\n'), _GrumpRun(code)) def testCallKwargs(self): code = textwrap.dedent("""\ def foo(a, b=2): print a, b foo(**{'a': 4})""") self.assertEqual((0, '4 2\n'), _GrumpRun(code)) testCompareLT = _MakeExprTest('1 < 2') testCompareLE = _MakeExprTest('7 <= 12') testCompareEq = _MakeExprTest('8 == 4') testCompareNE = _MakeExprTest('9 != 5') testCompareGE = _MakeExprTest('3 >= 2') testCompareGT = _MakeExprTest('2 > 6') testCompareLTLT = _MakeExprTest('3 < 6 < 9') testCompareLTEq = _MakeExprTest('3 < 6 == 9') testCompareLTGE = _MakeExprTest('3 < 6 >= -2') testCompareGTEq = _MakeExprTest('88 > 12 == 12') testCompareInStr = _MakeExprTest('"1" in "abc"') testCompareInTuple = _MakeExprTest('1 in (1, 2, 3)') testCompareNotInTuple = _MakeExprTest('10 < 12 not in (1, 2, 3)') testDictEmpty = _MakeLiteralTest('{}') testDictNonEmpty = _MakeLiteralTest("{'foo': 42, 'bar': 43}") testSetNonEmpty = _MakeLiteralTest("{'foo', 'bar'}", "set(['foo', 'bar'])") testDictCompFor = _MakeExprTest('{x: str(x) for x in range(3)}') testDictCompForIf = _MakeExprTest( '{x: 3 * x for x in range(10) if x % 3 == 0}') testDictCompForFor = _MakeExprTest( '{x: y for x in range(3) for y in range(x)}') testGeneratorExpFor = _MakeExprTest('tuple(int(x) for x in "123")') testGeneratorExpForIf = _MakeExprTest( 'tuple(x / 3 for x in range(10) if x % 3)') testGeneratorExprForFor = _MakeExprTest( 'tuple(x + y for x in range(3) for y in range(x + 2))') testIfExpr = _MakeExprTest('1 if True else 0') testIfExprCompound = _MakeExprTest('42 if "ab" == "a" + "b" else 24') testIfExprNested = _MakeExprTest( '"foo" if "" else "bar" if 0 else "baz"') testLambda = _MakeExprTest('(lambda: 123)()') testLambda = _MakeExprTest('(lambda a, b: (a, b))("foo", "bar")') testLambda = _MakeExprTest('(lambda a, b=3: (a, b))("foo")') testLambda = _MakeExprTest('(lambda *args: args)(1, 2, 3)') testLambda = _MakeExprTest('(lambda **kwargs: kwargs)(x="foo", y="bar")') testListEmpty = _MakeLiteralTest('[]') testListNonEmpty = _MakeLiteralTest('[1, 2]') testListCompFor = _MakeExprTest('[int(x) for x in "123"]') testListCompForIf = _MakeExprTest('[x / 3 for x in range(10) if x % 3]') testListCompForFor = _MakeExprTest( '[x + y for x in range(3) for y in range(x + 2)]') def testNameGlobal(self): code = textwrap.dedent("""\ foo = 123 assert foo == 123""") self.assertEqual((0, ''), _GrumpRun(code)) def testNameLocal(self): code = textwrap.dedent("""\ def foo(): bar = 'abc' assert bar == 'abc' foo()""") self.assertEqual((0, ''), _GrumpRun(code)) testNumInt = _MakeLiteralTest('42') testNumLong = _MakeLiteralTest('42L') testNumIntLarge = _MakeLiteralTest('12345678901234567890', '12345678901234567890L') testNumFloat = _MakeLiteralTest('102.1') testNumFloatOnlyDecimal = _MakeLiteralTest('.5', '0.5') testNumFloatNoDecimal = _MakeLiteralTest('5.', '5.0') testNumFloatSci = _MakeLiteralTest('1e6', '1000000.0') testNumFloatSciCap = _MakeLiteralTest('1E6', '1000000.0') testNumFloatSciCapPlus = _MakeLiteralTest('1E+6', '1000000.0') testNumFloatSciMinus = _MakeLiteralTest('1e-06') testNumComplex = _MakeLiteralTest('3j') testSubscriptDictStr = _MakeExprTest('{"foo": 42}["foo"]') testSubscriptListInt = _MakeExprTest('[1, 2, 3][2]') testSubscriptTupleSliceStart = _MakeExprTest('(1, 2, 3)[2:]') testSubscriptTupleSliceStartStop = _MakeExprTest('(1, 2, 3)[10:11]') testSubscriptTupleSliceStartStep = _MakeExprTest('(1, 2, 3, 4, 5, 6)[-2::-2]') testSubscriptStartStop = _MakeSliceTest('2:3', 'slice(2, 3, None)') testSubscriptMultiDim = _MakeSliceTest('1,2,3', '(1, 2, 3)') testSubscriptStartStopObjects = _MakeSliceTest( 'True:False', 'slice(True, False, None)') testSubscriptMultiDimSlice = _MakeSliceTest( "'foo','bar':'baz':'qux'", "('foo', slice('bar', 'baz', 'qux'))") testStrEmpty = _MakeLiteralTest("''") testStrAscii = _MakeLiteralTest("'abc'") testStrUtf8 = _MakeLiteralTest(r"'\tfoo\n\xcf\x80'") testStrQuoted = _MakeLiteralTest('\'"foo"\'', '\'"foo"\'') testStrUtf16 = _MakeLiteralTest("u'\\u0432\\u043e\\u043b\\u043d'") testTupleEmpty = _MakeLiteralTest('()') testTupleNonEmpty = _MakeLiteralTest('(1, 2, 3)') testUnaryOpNot = _MakeExprTest('not True') testUnaryOpInvert = _MakeExprTest('~4') testUnaryOpPos = _MakeExprTest('+4') def _MakeModuleBlock(): return block.ModuleBlock(None, '__main__', '', '', imputil.FutureFeatures()) def _ParseExpr(expr): return pythonparser.parse(expr).body[0].value def _ParseAndVisitExpr(expr): visitor = stmt.StatementVisitor(_MakeModuleBlock()) visitor.visit_expr(_ParseExpr(expr)) return visitor.writer.getvalue() def _GrumpRun(cmd): p = subprocess.Popen(['grumprun'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, _ = p.communicate(cmd) return p.returncode, out if __name__ == '__main__': shard_test.main() ================================================ FILE: compiler/imputil.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Functionality for importing modules in Grumpy.""" from __future__ import unicode_literals import collections import functools import os import os.path from grumpy.compiler import util from grumpy import pythonparser from grumpy.pythonparser import algorithm from grumpy.pythonparser import ast _NATIVE_MODULE_PREFIX = '__go__/' class Import(object): """Represents a single module import and all its associated bindings. Each import pertains to a single module that is imported. Thus one import statement may produce multiple Import objects. E.g. "import foo, bar" makes an Import object for module foo and another one for module bar. """ Binding = collections.namedtuple('Binding', ('bind_type', 'alias', 'value')) MODULE = "" MEMBER = "" def __init__(self, name, script=None, is_native=False): self.name = name self.script = script self.is_native = is_native self.bindings = [] def add_binding(self, bind_type, alias, value): self.bindings.append(Import.Binding(bind_type, alias, value)) class Importer(algorithm.Visitor): """Visits import nodes and produces corresponding Import objects.""" # pylint: disable=invalid-name,missing-docstring,no-init def __init__(self, gopath, modname, script, absolute_import): self.pathdirs = [] if gopath: self.pathdirs.extend(os.path.join(d, 'src', '__python__') for d in gopath.split(os.pathsep)) dirname, basename = os.path.split(script) if basename == '__init__.py': self.package_dir = dirname self.package_name = modname elif (modname.find('.') != -1 and os.path.isfile(os.path.join(dirname, '__init__.py'))): self.package_dir = dirname self.package_name = modname[:modname.rfind('.')] else: self.package_dir = None self.package_name = None self.absolute_import = absolute_import def generic_visit(self, node): raise ValueError('Import cannot visit {} node'.format(type(node).__name__)) def visit_Import(self, node): imports = [] for alias in node.names: if alias.name.startswith(_NATIVE_MODULE_PREFIX): imp = Import(alias.name, is_native=True) asname = alias.asname if alias.asname else alias.name.split('/')[-1] imp.add_binding(Import.MODULE, asname, 0) else: imp = self._resolve_import(node, alias.name) if alias.asname: imp.add_binding(Import.MODULE, alias.asname, imp.name.count('.')) else: parts = alias.name.split('.') imp.add_binding(Import.MODULE, parts[0], imp.name.count('.') - len(parts) + 1) imports.append(imp) return imports def visit_ImportFrom(self, node): if any(a.name == '*' for a in node.names): raise util.ImportError(node, 'wildcard member import is not implemented') if not node.level and node.module == '__future__': return [] if not node.level and node.module.startswith(_NATIVE_MODULE_PREFIX): imp = Import(node.module, is_native=True) for alias in node.names: asname = alias.asname or alias.name imp.add_binding(Import.MEMBER, asname, alias.name) return [imp] imports = [] if not node.module: # Import of the form 'from .. import foo, bar'. All named imports must be # modules, not module members. for alias in node.names: imp = self._resolve_relative_import(node.level, node, alias.name) imp.add_binding(Import.MODULE, alias.asname or alias.name, imp.name.count('.')) imports.append(imp) return imports member_imp = None for alias in node.names: asname = alias.asname or alias.name if node.level: resolver = functools.partial(self._resolve_relative_import, node.level) else: resolver = self._resolve_import try: imp = resolver(node, '{}.{}'.format(node.module, alias.name)) except util.ImportError: # A member (not a submodule) is being imported, so bind it. if not member_imp: member_imp = resolver(node, node.module) imports.append(member_imp) member_imp.add_binding(Import.MEMBER, asname, alias.name) else: # Imported name is a submodule within a package, so bind that module. imp.add_binding(Import.MODULE, asname, imp.name.count('.')) imports.append(imp) return imports def _resolve_import(self, node, modname): if not self.absolute_import and self.package_dir: script = find_script(self.package_dir, modname) if script: return Import('{}.{}'.format(self.package_name, modname), script) for dirname in self.pathdirs: script = find_script(dirname, modname) if script: return Import(modname, script) raise util.ImportError(node, 'no such module: {}'.format(modname)) def _resolve_relative_import(self, level, node, modname): if not self.package_dir: raise util.ImportError(node, 'attempted relative import in non-package') uplevel = level - 1 if uplevel > self.package_name.count('.'): raise util.ImportError( node, 'attempted relative import beyond toplevel package') dirname = os.path.normpath(os.path.join( self.package_dir, *(['..'] * uplevel))) script = find_script(dirname, modname) if not script: raise util.ImportError(node, 'no such module: {}'.format(modname)) parts = self.package_name.split('.') return Import('.'.join(parts[:len(parts)-uplevel]) + '.' + modname, script) class _ImportCollector(algorithm.Visitor): # pylint: disable=invalid-name def __init__(self, importer, future_node): self.importer = importer self.future_node = future_node self.imports = [] def visit_Import(self, node): self.imports.extend(self.importer.visit(node)) def visit_ImportFrom(self, node): if node.module == '__future__': if node != self.future_node: raise util.LateFutureError(node) return self.imports.extend(self.importer.visit(node)) def collect_imports(modname, script, gopath): with open(script) as py_file: py_contents = py_file.read() mod = pythonparser.parse(py_contents) future_node, future_features = parse_future_features(mod) importer = Importer(gopath, modname, script, future_features.absolute_import) collector = _ImportCollector(importer, future_node) collector.visit(mod) return collector.imports def calculate_transitive_deps(modname, script, gopath): """Determines all modules that script transitively depends upon.""" deps = set() def calc(modname, script): if modname in deps: return deps.add(modname) for imp in collect_imports(modname, script, gopath): if imp.is_native: deps.add(imp.name) continue parts = imp.name.split('.') calc(imp.name, imp.script) if len(parts) == 1: continue # For submodules, the parent packages are also deps. package_dir, filename = os.path.split(imp.script) if filename == '__init__.py': package_dir = os.path.dirname(package_dir) for i in xrange(len(parts) - 1, 0, -1): modname = '.'.join(parts[:i]) script = os.path.join(package_dir, '__init__.py') calc(modname, script) package_dir = os.path.dirname(package_dir) calc(modname, script) deps.remove(modname) return deps def find_script(dirname, name): prefix = os.path.join(dirname, name.replace('.', os.sep)) script = prefix + '.py' if os.path.isfile(script): return script script = os.path.join(prefix, '__init__.py') if os.path.isfile(script): return script return None _FUTURE_FEATURES = ( 'absolute_import', 'division', 'print_function', 'unicode_literals', ) _IMPLEMENTED_FUTURE_FEATURES = ( 'absolute_import', 'print_function', 'unicode_literals' ) # These future features are already in the language proper as of 2.6, so # importing them via __future__ has no effect. _REDUNDANT_FUTURE_FEATURES = ('generators', 'with_statement', 'nested_scopes') class FutureFeatures(object): """Spec for future feature flags imported by a module.""" def __init__(self, absolute_import=False, division=False, print_function=False, unicode_literals=False): self.absolute_import = absolute_import self.division = division self.print_function = print_function self.unicode_literals = unicode_literals def _make_future_features(node): """Processes a future import statement, returning set of flags it defines.""" assert isinstance(node, ast.ImportFrom) assert node.module == '__future__' features = FutureFeatures() for alias in node.names: name = alias.name if name in _FUTURE_FEATURES: if name not in _IMPLEMENTED_FUTURE_FEATURES: msg = 'future feature {} not yet implemented by grumpy'.format(name) raise util.ParseError(node, msg) setattr(features, name, True) elif name == 'braces': raise util.ParseError(node, 'not a chance') elif name not in _REDUNDANT_FUTURE_FEATURES: msg = 'future feature {} is not defined'.format(name) raise util.ParseError(node, msg) return features def parse_future_features(mod): """Accumulates a set of flags for the compiler __future__ imports.""" assert isinstance(mod, ast.Module) found_docstring = False for node in mod.body: if isinstance(node, ast.ImportFrom): if node.module == '__future__': return node, _make_future_features(node) break elif isinstance(node, ast.Expr) and not found_docstring: if not isinstance(node.value, ast.Str): break found_docstring = True else: break return None, FutureFeatures() ================================================ FILE: compiler/imputil_test.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests ImportVisitor and related classes.""" from __future__ import unicode_literals import copy import os import shutil import tempfile import textwrap import unittest from grumpy.compiler import imputil from grumpy.compiler import util from grumpy import pythonparser class ImportVisitorTest(unittest.TestCase): _PATH_SPEC = { 'foo.py': None, 'qux.py': None, 'bar/': { 'fred/': { '__init__.py': None, 'quux.py': None, }, '__init__.py': None, 'baz.py': None, 'foo.py': None, }, 'baz.py': None, } def setUp(self): self.rootdir = tempfile.mkdtemp() self.pydir = os.path.join(self.rootdir, 'src', '__python__') self._materialize_tree( self.rootdir, {'src/': {'__python__/': self._PATH_SPEC}}) foo_script = os.path.join(self.rootdir, 'foo.py') self.importer = imputil.Importer(self.rootdir, 'foo', foo_script, False) bar_script = os.path.join(self.pydir, 'bar', '__init__.py') self.bar_importer = imputil.Importer( self.rootdir, 'bar', bar_script, False) fred_script = os.path.join(self.pydir, 'bar', 'fred', '__init__.py') self.fred_importer = imputil.Importer( self.rootdir, 'bar.fred', fred_script, False) self.foo_import = imputil.Import( 'foo', os.path.join(self.pydir, 'foo.py')) self.qux_import = imputil.Import( 'qux', os.path.join(self.pydir, 'qux.py')) self.bar_import = imputil.Import( 'bar', os.path.join(self.pydir, 'bar/__init__.py')) self.fred_import = imputil.Import( 'bar.fred', os.path.join(self.pydir, 'bar/fred/__init__.py')) self.quux_import = imputil.Import( 'bar.fred.quux', os.path.join(self.pydir, 'bar/fred/quux.py')) self.baz2_import = imputil.Import( 'bar.baz', os.path.join(self.pydir, 'bar/baz.py')) self.foo2_import = imputil.Import( 'bar.foo', os.path.join(self.pydir, 'bar/foo.py')) self.baz_import = imputil.Import( 'baz', os.path.join(self.pydir, 'baz.py')) def tearDown(self): shutil.rmtree(self.rootdir) def testImportEmptyPath(self): importer = imputil.Importer(None, 'foo', 'foo.py', False) self.assertRaises(util.ImportError, importer.visit, pythonparser.parse('import bar').body[0]) def testImportTopLevelModule(self): imp = copy.deepcopy(self.qux_import) imp.add_binding(imputil.Import.MODULE, 'qux', 0) self._check_imports('import qux', [imp]) def testImportTopLevelPackage(self): imp = copy.deepcopy(self.bar_import) imp.add_binding(imputil.Import.MODULE, 'bar', 0) self._check_imports('import bar', [imp]) def testImportPackageModuleAbsolute(self): imp = copy.deepcopy(self.baz2_import) imp.add_binding(imputil.Import.MODULE, 'bar', 0) self._check_imports('import bar.baz', [imp]) def testImportFromSubModule(self): imp = copy.deepcopy(self.baz2_import) imp.add_binding(imputil.Import.MODULE, 'baz', 1) self._check_imports('from bar import baz', [imp]) def testImportPackageModuleRelative(self): imp = copy.deepcopy(self.baz2_import) imp.add_binding(imputil.Import.MODULE, 'baz', 1) got = self.bar_importer.visit(pythonparser.parse('import baz').body[0]) self._assert_imports_equal([imp], got) def testImportPackageModuleRelativeFromSubModule(self): imp = copy.deepcopy(self.baz2_import) imp.add_binding(imputil.Import.MODULE, 'baz', 1) foo_script = os.path.join(self.pydir, 'bar', 'foo.py') importer = imputil.Importer(self.rootdir, 'bar.foo', foo_script, False) got = importer.visit(pythonparser.parse('import baz').body[0]) self._assert_imports_equal([imp], got) def testImportPackageModuleAbsoluteImport(self): imp = copy.deepcopy(self.baz_import) imp.add_binding(imputil.Import.MODULE, 'baz', 0) bar_script = os.path.join(self.pydir, 'bar', '__init__.py') importer = imputil.Importer(self.rootdir, 'bar', bar_script, True) got = importer.visit(pythonparser.parse('import baz').body[0]) self._assert_imports_equal([imp], got) def testImportMultiple(self): imp1 = copy.deepcopy(self.foo_import) imp1.add_binding(imputil.Import.MODULE, 'foo', 0) imp2 = copy.deepcopy(self.baz2_import) imp2.add_binding(imputil.Import.MODULE, 'bar', 0) self._check_imports('import foo, bar.baz', [imp1, imp2]) def testImportAs(self): imp = copy.deepcopy(self.foo_import) imp.add_binding(imputil.Import.MODULE, 'bar', 0) self._check_imports('import foo as bar', [imp]) def testImportFrom(self): imp = copy.deepcopy(self.baz2_import) imp.add_binding(imputil.Import.MODULE, 'baz', 1) self._check_imports('from bar import baz', [imp]) def testImportFromMember(self): imp = copy.deepcopy(self.foo_import) imp.add_binding(imputil.Import.MEMBER, 'bar', 'bar') self._check_imports('from foo import bar', [imp]) def testImportFromMultiple(self): imp1 = copy.deepcopy(self.baz2_import) imp1.add_binding(imputil.Import.MODULE, 'baz', 1) imp2 = copy.deepcopy(self.foo2_import) imp2.add_binding(imputil.Import.MODULE, 'foo', 1) self._check_imports('from bar import baz, foo', [imp1, imp2]) def testImportFromMixedMembers(self): imp1 = copy.deepcopy(self.bar_import) imp1.add_binding(imputil.Import.MEMBER, 'qux', 'qux') imp2 = copy.deepcopy(self.baz2_import) imp2.add_binding(imputil.Import.MODULE, 'baz', 1) self._check_imports('from bar import qux, baz', [imp1, imp2]) def testImportFromAs(self): imp = copy.deepcopy(self.baz2_import) imp.add_binding(imputil.Import.MODULE, 'qux', 1) self._check_imports('from bar import baz as qux', [imp]) def testImportFromAsMembers(self): imp = copy.deepcopy(self.foo_import) imp.add_binding(imputil.Import.MEMBER, 'baz', 'bar') self._check_imports('from foo import bar as baz', [imp]) def testImportFromWildcardRaises(self): self.assertRaises(util.ImportError, self.importer.visit, pythonparser.parse('from foo import *').body[0]) def testImportFromFuture(self): self._check_imports('from __future__ import print_function', []) def testImportFromNative(self): imp = imputil.Import('__go__/fmt', is_native=True) imp.add_binding(imputil.Import.MEMBER, 'Printf', 'Printf') self._check_imports('from "__go__/fmt" import Printf', [imp]) def testImportFromNativeMultiple(self): imp = imputil.Import('__go__/fmt', is_native=True) imp.add_binding(imputil.Import.MEMBER, 'Printf', 'Printf') imp.add_binding(imputil.Import.MEMBER, 'Println', 'Println') self._check_imports('from "__go__/fmt" import Printf, Println', [imp]) def testImportFromNativeAs(self): imp = imputil.Import('__go__/fmt', is_native=True) imp.add_binding(imputil.Import.MEMBER, 'foo', 'Printf') self._check_imports('from "__go__/fmt" import Printf as foo', [imp]) def testRelativeImportNonPackage(self): self.assertRaises(util.ImportError, self.importer.visit, pythonparser.parse('from . import bar').body[0]) def testRelativeImportBeyondTopLevel(self): self.assertRaises(util.ImportError, self.bar_importer.visit, pythonparser.parse('from .. import qux').body[0]) def testRelativeModuleNoExist(self): self.assertRaises(util.ImportError, self.bar_importer.visit, pythonparser.parse('from . import qux').body[0]) def testRelativeModule(self): imp = copy.deepcopy(self.foo2_import) imp.add_binding(imputil.Import.MODULE, 'foo', 1) node = pythonparser.parse('from . import foo').body[0] self._assert_imports_equal([imp], self.bar_importer.visit(node)) def testRelativeModuleFromSubModule(self): imp = copy.deepcopy(self.foo2_import) imp.add_binding(imputil.Import.MODULE, 'foo', 1) baz_script = os.path.join(self.pydir, 'bar', 'baz.py') importer = imputil.Importer(self.rootdir, 'bar.baz', baz_script, False) node = pythonparser.parse('from . import foo').body[0] self._assert_imports_equal([imp], importer.visit(node)) def testRelativeModuleMember(self): imp = copy.deepcopy(self.foo2_import) imp.add_binding(imputil.Import.MEMBER, 'qux', 'qux') node = pythonparser.parse('from .foo import qux').body[0] self._assert_imports_equal([imp], self.bar_importer.visit(node)) def testRelativeModuleMemberMixed(self): imp1 = copy.deepcopy(self.fred_import) imp1.add_binding(imputil.Import.MEMBER, 'qux', 'qux') imp2 = copy.deepcopy(self.quux_import) imp2.add_binding(imputil.Import.MODULE, 'quux', 2) node = pythonparser.parse('from .fred import qux, quux').body[0] self._assert_imports_equal([imp1, imp2], self.bar_importer.visit(node)) def testRelativeUpLevel(self): imp = copy.deepcopy(self.foo2_import) imp.add_binding(imputil.Import.MODULE, 'foo', 1) node = pythonparser.parse('from .. import foo').body[0] self._assert_imports_equal([imp], self.fred_importer.visit(node)) def testRelativeUpLevelMember(self): imp = copy.deepcopy(self.foo2_import) imp.add_binding(imputil.Import.MEMBER, 'qux', 'qux') node = pythonparser.parse('from ..foo import qux').body[0] self._assert_imports_equal([imp], self.fred_importer.visit(node)) def _check_imports(self, stmt, want): got = self.importer.visit(pythonparser.parse(stmt).body[0]) self._assert_imports_equal(want, got) def _assert_imports_equal(self, want, got): self.assertEqual([imp.__dict__ for imp in want], [imp.__dict__ for imp in got]) def _materialize_tree(self, dirname, spec): for name, sub_spec in spec.iteritems(): if name.endswith('/'): subdir = os.path.join(dirname, name[:-1]) os.mkdir(subdir) self._materialize_tree(subdir, sub_spec) else: with open(os.path.join(dirname, name), 'w'): pass class MakeFutureFeaturesTest(unittest.TestCase): def testImportFromFuture(self): testcases = [ ('from __future__ import print_function', imputil.FutureFeatures(print_function=True)), ('from __future__ import generators', imputil.FutureFeatures()), ('from __future__ import generators, print_function', imputil.FutureFeatures(print_function=True)), ] for tc in testcases: source, want = tc mod = pythonparser.parse(textwrap.dedent(source)) node = mod.body[0] got = imputil._make_future_features(node) # pylint: disable=protected-access self.assertEqual(want.__dict__, got.__dict__) def testImportFromFutureParseError(self): testcases = [ # NOTE: move this group to testImportFromFuture as they are implemented # by grumpy ('from __future__ import division', r'future feature \w+ not yet implemented'), ('from __future__ import braces', 'not a chance'), ('from __future__ import nonexistant_feature', r'future feature \w+ is not defined'), ] for tc in testcases: source, want_regexp = tc mod = pythonparser.parse(source) node = mod.body[0] self.assertRaisesRegexp(util.ParseError, want_regexp, imputil._make_future_features, node) # pylint: disable=protected-access class ParseFutureFeaturesTest(unittest.TestCase): def testFutureFeatures(self): testcases = [ ('from __future__ import print_function', imputil.FutureFeatures(print_function=True)), ("""\ "module docstring" from __future__ import print_function """, imputil.FutureFeatures(print_function=True)), ("""\ "module docstring" from __future__ import print_function, with_statement from __future__ import nested_scopes """, imputil.FutureFeatures(print_function=True)), ('from __future__ import absolute_import', imputil.FutureFeatures(absolute_import=True)), ('from __future__ import absolute_import, print_function', imputil.FutureFeatures(absolute_import=True, print_function=True)), ('foo = 123\nfrom __future__ import print_function', imputil.FutureFeatures()), ('import os\nfrom __future__ import print_function', imputil.FutureFeatures()), ] for tc in testcases: source, want = tc mod = pythonparser.parse(textwrap.dedent(source)) _, got = imputil.parse_future_features(mod) self.assertEqual(want.__dict__, got.__dict__) def testUnimplementedFutureRaises(self): mod = pythonparser.parse('from __future__ import division') msg = 'future feature division not yet implemented by grumpy' self.assertRaisesRegexp(util.ParseError, msg, imputil.parse_future_features, mod) def testUndefinedFutureRaises(self): mod = pythonparser.parse('from __future__ import foo') self.assertRaisesRegexp( util.ParseError, 'future feature foo is not defined', imputil.parse_future_features, mod) if __name__ == '__main__': unittest.main() ================================================ FILE: compiler/shard_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Wrapper for unit tests that loads a subset of all test methods.""" from __future__ import unicode_literals import argparse import random import re import sys import unittest class _ShardAction(argparse.Action): def __call__(self, parser, args, values, option_string=None): match = re.match(r'(\d+)of(\d+)$', values) if not match: raise argparse.ArgumentError(self, 'bad shard spec: {}'.format(values)) shard = int(match.group(1)) count = int(match.group(2)) if shard < 1 or count < 1 or shard > count: raise argparse.ArgumentError(self, 'bad shard spec: {}'.format(values)) setattr(args, self.dest, (shard, count)) class _ShardTestLoader(unittest.TestLoader): def __init__(self, shard, count): super(_ShardTestLoader, self).__init__() self.shard = shard self.count = count def getTestCaseNames(self, test_case_cls): names = super(_ShardTestLoader, self).getTestCaseNames(test_case_cls) state = random.getstate() random.seed(self.count) random.shuffle(names) random.setstate(state) n = len(names) # self.shard is one-based. return names[(self.shard - 1) * n / self.count:self.shard * n / self.count] class _ShardTestRunner(object): def run(self, test): result = unittest.TestResult() unittest.registerResult(result) test(result) for kind, errors in [('FAIL', result.failures), ('ERROR', result.errors)]: for test, err in errors: sys.stderr.write('{} {}\n{}'.format(test, kind, err)) return result def main(): parser = argparse.ArgumentParser() parser.add_argument('--shard', default=(1, 1), action=_ShardAction) parser.add_argument('unittest_args', nargs='*') args = parser.parse_args() unittest.main(argv=[sys.argv[0]] + args.unittest_args, testLoader=_ShardTestLoader(*args.shard), testRunner=_ShardTestRunner) ================================================ FILE: compiler/stmt.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Visitor class for traversing Python statements.""" from __future__ import unicode_literals import string import textwrap from grumpy.compiler import block from grumpy.compiler import expr from grumpy.compiler import expr_visitor from grumpy.compiler import imputil from grumpy.compiler import util from grumpy.pythonparser import algorithm from grumpy.pythonparser import ast _NATIVE_TYPE_PREFIX = 'type_' # Partial list of known vcs for go module import # Full list can be found at https://golang.org/src/cmd/go/vcs.go # TODO: Use official vcs.go module instead of partial list _KNOWN_VCS = [ 'golang.org', 'github.com', 'bitbucket.org', 'git.apache.org', 'git.openstack.org', 'launchpad.net' ] _nil_expr = expr.nil_expr class StatementVisitor(algorithm.Visitor): """Outputs Go statements to a Writer for the given Python nodes.""" # pylint: disable=invalid-name,missing-docstring def __init__(self, block_, future_node=None): self.block = block_ self.future_node = future_node self.writer = util.Writer() self.expr_visitor = expr_visitor.ExprVisitor(self) def generic_visit(self, node): msg = 'node not yet implemented: {}'.format(type(node).__name__) raise util.ParseError(node, msg) def visit_expr(self, node): return self.expr_visitor.visit(node) def visit_Assert(self, node): self._write_py_context(node.lineno) # TODO: Only evaluate msg if cond is false. with self.visit_expr(node.msg) if node.msg else _nil_expr as msg,\ self.visit_expr(node.test) as cond: self.writer.write_checked_call1( 'πg.Assert(πF, {}, {})', cond.expr, msg.expr) def visit_AugAssign(self, node): op_type = type(node.op) if op_type not in StatementVisitor._AUG_ASSIGN_TEMPLATES: fmt = 'augmented assignment op not implemented: {}' raise util.ParseError(node, fmt.format(op_type.__name__)) self._write_py_context(node.lineno) with self.visit_expr(node.target) as target,\ self.visit_expr(node.value) as value,\ self.block.alloc_temp() as temp: self.writer.write_checked_call2( temp, StatementVisitor._AUG_ASSIGN_TEMPLATES[op_type], lhs=target.expr, rhs=value.expr) self._assign_target(node.target, temp.expr) def visit_Assign(self, node): self._write_py_context(node.lineno) with self.visit_expr(node.value) as value: for target in node.targets: self._tie_target(target, value.expr) def visit_Break(self, node): if not self.block.loop_stack: raise util.ParseError(node, "'break' not in loop") self._write_py_context(node.lineno) self.writer.write_tmpl(textwrap.dedent("""\ $breakvar = true continue"""), breakvar=self.block.top_loop().breakvar.name) def visit_ClassDef(self, node): # Since we only care about global vars, we end up throwing away the locals # collected by BlockVisitor. But use it anyway since it buys us detection of # assignment to vars that are later declared global. block_visitor = block.BlockVisitor() for child in node.body: block_visitor.visit(child) global_vars = {v.name for v in block_visitor.vars.values() if v.type == block.Var.TYPE_GLOBAL} # Visit all the statements inside body of the class definition. body_visitor = StatementVisitor(block.ClassBlock( self.block, node.name, global_vars), self.future_node) # Indent so that the function body is aligned with the goto labels. with body_visitor.writer.indent_block(): body_visitor._visit_each(node.body) # pylint: disable=protected-access self._write_py_context(node.lineno) with self.block.alloc_temp('*πg.Dict') as cls, \ self.block.alloc_temp() as mod_name, \ self.block.alloc_temp('[]*πg.Object') as bases, \ self.block.alloc_temp() as meta: self.writer.write('{} = make([]*πg.Object, {})'.format( bases.expr, len(node.bases))) for i, b in enumerate(node.bases): with self.visit_expr(b) as b: self.writer.write('{}[{}] = {}'.format(bases.expr, i, b.expr)) self.writer.write('{} = πg.NewDict()'.format(cls.name)) self.writer.write_checked_call2( mod_name, 'πF.Globals().GetItem(πF, {}.ToObject())', self.block.root.intern('__name__')) self.writer.write_checked_call1( '{}.SetItem(πF, {}.ToObject(), {})', cls.expr, self.block.root.intern('__module__'), mod_name.expr) tmpl = textwrap.dedent(""" _, πE = πg.NewCode($name, $filename, nil, 0, func(πF *πg.Frame, _ []*πg.Object) (*πg.Object, *πg.BaseException) { \tπClass := $cls \t_ = πClass""") self.writer.write_tmpl(tmpl, name=util.go_str(node.name), filename=util.go_str(self.block.root.filename), cls=cls.expr) with self.writer.indent_block(): self.writer.write_temp_decls(body_visitor.block) self.writer.write_block(body_visitor.block, body_visitor.writer.getvalue()) self.writer.write('return nil, nil') tmpl = textwrap.dedent("""\ }).Eval(πF, πF.Globals(), nil, nil) if πE != nil { \tcontinue } if $meta, πE = $cls.GetItem(πF, $metaclass_str.ToObject()); πE != nil { \tcontinue } if $meta == nil { \t$meta = πg.TypeType.ToObject() }""") self.writer.write_tmpl( tmpl, meta=meta.name, cls=cls.expr, metaclass_str=self.block.root.intern('__metaclass__')) with self.block.alloc_temp() as type_: type_expr = ('{}.Call(πF, []*πg.Object{{πg.NewStr({}).ToObject(), ' 'πg.NewTuple({}...).ToObject(), {}.ToObject()}}, nil)') self.writer.write_checked_call2( type_, type_expr, meta.expr, util.go_str(node.name), bases.expr, cls.expr) self.block.bind_var(self.writer, node.name, type_.expr) def visit_Continue(self, node): if not self.block.loop_stack: raise util.ParseError(node, "'continue' not in loop") self._write_py_context(node.lineno) self.writer.write('continue') def visit_Delete(self, node): self._write_py_context(node.lineno) for target in node.targets: if isinstance(target, ast.Attribute): with self.visit_expr(target.value) as t: self.writer.write_checked_call1( 'πg.DelAttr(πF, {}, {})', t.expr, self.block.root.intern(target.attr)) elif isinstance(target, ast.Name): self.block.del_var(self.writer, target.id) elif isinstance(target, ast.Subscript): with self.visit_expr(target.value) as t,\ self.visit_expr(target.slice) as index: self.writer.write_checked_call1('πg.DelItem(πF, {}, {})', t.expr, index.expr) else: msg = 'del target not implemented: {}'.format(type(target).__name__) raise util.ParseError(node, msg) def visit_Expr(self, node): self._write_py_context(node.lineno) self.visit_expr(node.value).free() def visit_For(self, node): with self.block.alloc_temp() as i: with self.visit_expr(node.iter) as iter_expr: self.writer.write_checked_call2(i, 'πg.Iter(πF, {})', iter_expr.expr) def testfunc(testvar): with self.block.alloc_temp() as n: self.writer.write_tmpl(textwrap.dedent("""\ if $n, πE = πg.Next(πF, $i); πE != nil { \tisStop, exc := πg.IsInstance(πF, πE.ToObject(), πg.StopIterationType.ToObject()) \tif exc != nil { \t\tπE = exc \t} else if isStop { \t\tπE = nil \t\tπF.RestoreExc(nil, nil) \t} \t$testvar = !isStop } else { \t$testvar = true"""), n=n.name, i=i.expr, testvar=testvar.name) with self.writer.indent_block(): self._tie_target(node.target, n.expr) self.writer.write('}') self._visit_loop(testfunc, node) def visit_FunctionDef(self, node): self._write_py_context(node.lineno + len(node.decorator_list)) func = self.visit_function_inline(node) self.block.bind_var(self.writer, node.name, func.expr) while node.decorator_list: decorator = node.decorator_list.pop() wrapped = ast.Name(id=node.name) decorated = ast.Call(func=decorator, args=[wrapped], keywords=[], starargs=None, kwargs=None) target = ast.Assign(targets=[wrapped], value=decorated, loc=node.loc) self.visit_Assign(target) def visit_Global(self, node): self._write_py_context(node.lineno) def visit_If(self, node): # Collect the nodes for each if/elif/else body and write the dispatching # switch statement. bodies = [] # An elif clause is represented as a single If node within the orelse # section of the previous If node. Thus this loop terminates once we are # done all the elif clauses at which time the orelse var will contain the # nodes (if any) for the else clause. orelse = [node] while len(orelse) == 1 and isinstance(orelse[0], ast.If): ifnode = orelse[0] with self.visit_expr(ifnode.test) as cond: label = self.block.genlabel() # We goto the body of the if statement instead of executing it inline # because the body itself may be a goto target and Go does not support # jumping to targets inside a block. with self.block.alloc_temp('bool') as is_true: self.writer.write_tmpl(textwrap.dedent("""\ if $is_true, πE = πg.IsTrue(πF, $cond); πE != nil { \tcontinue } if $is_true { \tgoto Label$label }"""), is_true=is_true.name, cond=cond.expr, label=label) bodies.append((label, ifnode.body, ifnode.lineno)) orelse = ifnode.orelse default_label = end_label = self.block.genlabel() if orelse: end_label = self.block.genlabel() # The else is not represented by ast and thus there is no lineno. bodies.append((default_label, orelse, None)) self.writer.write('goto Label{}'.format(default_label)) # Write the body of each clause. for label, body, lineno in bodies: if lineno: self._write_py_context(lineno) self.writer.write_label(label) self._visit_each(body) self.writer.write('goto Label{}'.format(end_label)) self.writer.write_label(end_label) def visit_Import(self, node): self._write_py_context(node.lineno) for imp in self.block.root.importer.visit(node): self._import_and_bind(imp) def visit_ImportFrom(self, node): self._write_py_context(node.lineno) if node.module == '__future__' and node != self.future_node: raise util.LateFutureError(node) for imp in self.block.root.importer.visit(node): self._import_and_bind(imp) def visit_Module(self, node): self._visit_each(node.body) def visit_Pass(self, node): self._write_py_context(node.lineno) def visit_Print(self, node): if self.block.root.future_features.print_function: raise util.ParseError(node, 'syntax error (print is not a keyword)') self._write_py_context(node.lineno) with self.block.alloc_temp('[]*πg.Object') as args: self.writer.write('{} = make([]*πg.Object, {})'.format( args.expr, len(node.values))) for i, v in enumerate(node.values): with self.visit_expr(v) as arg: self.writer.write('{}[{}] = {}'.format(args.expr, i, arg.expr)) self.writer.write_checked_call1('πg.Print(πF, {}, {})', args.expr, 'true' if node.nl else 'false') def visit_Raise(self, node): with self.visit_expr(node.exc) if node.exc else _nil_expr as t,\ self.visit_expr(node.inst) if node.inst else _nil_expr as inst,\ self.visit_expr(node.tback) if node.tback else _nil_expr as tb: if node.inst: assert node.exc, 'raise had inst but no type' if node.tback: assert node.inst, 'raise had tback but no inst' self._write_py_context(node.lineno) self.writer.write('πE = πF.Raise({}, {}, {})'.format( t.expr, inst.expr, tb.expr)) self.writer.write('continue') def visit_Return(self, node): assert isinstance(self.block, block.FunctionBlock) self._write_py_context(node.lineno) if self.block.is_generator and node.value: raise util.ParseError(node, 'returning a value in a generator function') if node.value: with self.visit_expr(node.value) as value: self.writer.write('πR = {}'.format(value.expr)) else: self.writer.write('πR = πg.None') self.writer.write('continue') def visit_Try(self, node): # The general structure generated by this method is shown below: # # checkpoints.Push(Except) # # Checkpoints.Pop() # # goto Finally # Except: # # Handler1: # # Checkpoints.Pop() // Finally # goto Finally # Handler2: # # Checkpoints.Pop() // Finally # goto Finally # ... # Finally: # # # The dispatch table maps the current exception to the appropriate handler # label according to the exception clauses. # Write the try body. self._write_py_context(node.lineno) finally_label = self.block.genlabel(is_checkpoint=bool(node.finalbody)) if node.finalbody: self.writer.write('πF.PushCheckpoint({})'.format(finally_label)) except_label = None if node.handlers: except_label = self.block.genlabel(is_checkpoint=True) self.writer.write('πF.PushCheckpoint({})'.format(except_label)) self._visit_each(node.body) if except_label: self.writer.write('πF.PopCheckpoint()') # except_label if node.orelse: self._visit_each(node.orelse) if node.finalbody: self.writer.write('πF.PopCheckpoint()') # finally_label self.writer.write('goto Label{}'.format(finally_label)) with self.block.alloc_temp('*πg.BaseException') as exc: if except_label: with self.block.alloc_temp('*πg.Traceback') as tb: self.writer.write_label(except_label) self.writer.write_tmpl(textwrap.dedent("""\ if πE == nil { continue } πE = nil $exc, $tb = πF.ExcInfo()"""), exc=exc.expr, tb=tb.expr) handler_labels = self._write_except_dispatcher( exc.expr, tb.expr, node.handlers) # Write the bodies of each of the except handlers. for handler_label, except_node in zip(handler_labels, node.handlers): self._write_except_block(handler_label, exc.expr, except_node) if node.finalbody: self.writer.write('πF.PopCheckpoint()') # finally_label self.writer.write('goto Label{}'.format(finally_label)) # Write the finally body. self.writer.write_label(finally_label) if node.finalbody: with self.block.alloc_temp('*πg.Traceback') as tb: self.writer.write('{}, {} = πF.RestoreExc(nil, nil)'.format( exc.expr, tb.expr)) self._visit_each(node.finalbody) self.writer.write_tmpl(textwrap.dedent("""\ if $exc != nil { \tπE = πF.Raise($exc.ToObject(), nil, $tb.ToObject()) \tcontinue } if πR != nil { \tcontinue }"""), exc=exc.expr, tb=tb.expr) def visit_While(self, node): self._write_py_context(node.lineno) def testfunc(testvar): with self.visit_expr(node.test) as cond: self.writer.write_checked_call2( testvar, 'πg.IsTrue(πF, {})', cond.expr) self._visit_loop(testfunc, node) def visit_With(self, node): assert len(node.items) == 1, 'multiple items in a with not yet supported' item = node.items[0] self._write_py_context(node.loc.line()) # mgr := EXPR with self.visit_expr(item.context_expr) as mgr,\ self.block.alloc_temp() as exit_func,\ self.block.alloc_temp() as value: # The code here has a subtle twist: It gets the exit function attribute # from the class, not from the object. This matches the pseudo code from # PEP 343 exactly, and is very close to what CPython actually does. (The # CPython implementation actually uses a special lookup which is performed # on the object, but skips the instance dictionary: see ceval.c and # lookup_maybe in typeobject.c.) # exit := type(mgr).__exit__ self.writer.write_checked_call2( exit_func, 'πg.GetAttr(πF, {}.Type().ToObject(), {}, nil)', mgr.expr, self.block.root.intern('__exit__')) # value := type(mgr).__enter__(mgr) self.writer.write_checked_call2( value, 'πg.GetAttr(πF, {}.Type().ToObject(), {}, nil)', mgr.expr, self.block.root.intern('__enter__')) self.writer.write_checked_call2( value, '{}.Call(πF, πg.Args{{{}}}, nil)', value.expr, mgr.expr) finally_label = self.block.genlabel(is_checkpoint=True) self.writer.write('πF.PushCheckpoint({})'.format(finally_label)) if item.optional_vars: self._tie_target(item.optional_vars, value.expr) self._visit_each(node.body) self.writer.write('πF.PopCheckpoint()') self.writer.write_label(finally_label) with self.block.alloc_temp() as swallow_exc,\ self.block.alloc_temp('bool') as swallow_exc_bool,\ self.block.alloc_temp('*πg.BaseException') as exc,\ self.block.alloc_temp('*πg.Traceback') as tb,\ self.block.alloc_temp('*πg.Type') as t: # temp := exit(mgr, *sys.exec_info()) tmpl = """\ $exc, $tb = nil, nil if πE != nil { \t$exc, $tb = πF.ExcInfo() } if $exc != nil { \t$t = $exc.Type() \tif $swallow_exc, πE = $exit_func.Call(πF, πg.Args{$mgr, $t.ToObject(), $exc.ToObject(), $tb.ToObject()}, nil); πE != nil { \t\tcontinue \t} } else { \tif $swallow_exc, πE = $exit_func.Call(πF, πg.Args{$mgr, πg.None, πg.None, πg.None}, nil); πE != nil { \t\tcontinue \t} } """ self.writer.write_tmpl( textwrap.dedent(tmpl), exc=exc.expr, tb=tb.expr, t=t.name, mgr=mgr.expr, exit_func=exit_func.expr, swallow_exc=swallow_exc.name) # if Exc != nil && swallow_exc != true { # Raise(nil, nil) # } self.writer.write_checked_call2( swallow_exc_bool, 'πg.IsTrue(πF, {})', swallow_exc.expr) self.writer.write_tmpl(textwrap.dedent("""\ if $exc != nil && $swallow_exc != true { \tπE = πF.Raise(nil, nil, nil) \tcontinue } if πR != nil { \tcontinue }"""), exc=exc.expr, swallow_exc=swallow_exc_bool.expr) def visit_function_inline(self, node): """Returns an GeneratedExpr for a function with the given body.""" # First pass collects the names of locals used in this function. Do this in # a separate pass so that we know whether to resolve a name as a local or a # global during the second pass. func_visitor = block.FunctionBlockVisitor(node) for child in node.body: func_visitor.visit(child) func_block = block.FunctionBlock(self.block, node.name, func_visitor.vars, func_visitor.is_generator) visitor = StatementVisitor(func_block, self.future_node) # Indent so that the function body is aligned with the goto labels. with visitor.writer.indent_block(): visitor._visit_each(node.body) # pylint: disable=protected-access result = self.block.alloc_temp() with self.block.alloc_temp('[]πg.Param') as func_args: args = node.args argc = len(args.args) self.writer.write('{} = make([]πg.Param, {})'.format( func_args.expr, argc)) # The list of defaults only contains args for which a default value is # specified so pad it with None to make it the same length as args. defaults = [None] * (argc - len(args.defaults)) + args.defaults for i, (a, d) in enumerate(zip(args.args, defaults)): with self.visit_expr(d) if d else expr.nil_expr as default: tmpl = '$args[$i] = πg.Param{Name: $name, Def: $default}' self.writer.write_tmpl(tmpl, args=func_args.expr, i=i, name=util.go_str(a.arg), default=default.expr) flags = [] if args.vararg: flags.append('πg.CodeFlagVarArg') if args.kwarg: flags.append('πg.CodeFlagKWArg') # The function object gets written to a temporary writer because we need # it as an expression that we subsequently bind to some variable. self.writer.write_tmpl( '$result = πg.NewFunction(πg.NewCode($name, $filename, $args, ' '$flags, func(πF *πg.Frame, πArgs []*πg.Object) ' '(*πg.Object, *πg.BaseException) {', result=result.name, name=util.go_str(node.name), filename=util.go_str(self.block.root.filename), args=func_args.expr, flags=' | '.join(flags) if flags else 0) with self.writer.indent_block(): for var in func_block.vars.values(): if var.type != block.Var.TYPE_GLOBAL: fmt = 'var {0} *πg.Object = {1}; _ = {0}' self.writer.write(fmt.format( util.adjust_local_name(var.name), var.init_expr)) self.writer.write_temp_decls(func_block) self.writer.write('var πR *πg.Object; _ = πR') self.writer.write('var πE *πg.BaseException; _ = πE') if func_block.is_generator: self.writer.write( 'return πg.NewGenerator(πF, func(πSent *πg.Object) ' '(*πg.Object, *πg.BaseException) {') with self.writer.indent_block(): self.writer.write_block(func_block, visitor.writer.getvalue()) self.writer.write('return nil, πE') self.writer.write('}).ToObject(), nil') else: self.writer.write_block(func_block, visitor.writer.getvalue()) self.writer.write(textwrap.dedent("""\ if πE != nil { \tπR = nil } else if πR == nil { \tπR = πg.None } return πR, πE""")) self.writer.write('}), πF.Globals()).ToObject()') return result _AUG_ASSIGN_TEMPLATES = { ast.Add: 'πg.IAdd(πF, {lhs}, {rhs})', ast.BitAnd: 'πg.IAnd(πF, {lhs}, {rhs})', ast.Div: 'πg.IDiv(πF, {lhs}, {rhs})', ast.FloorDiv: 'πg.IFloorDiv(πF, {lhs}, {rhs})', ast.LShift: 'πg.ILShift(πF, {lhs}, {rhs})', ast.Mod: 'πg.IMod(πF, {lhs}, {rhs})', ast.Mult: 'πg.IMul(πF, {lhs}, {rhs})', ast.BitOr: 'πg.IOr(πF, {lhs}, {rhs})', ast.Pow: 'πg.IPow(πF, {lhs}, {rhs})', ast.RShift: 'πg.IRShift(πF, {lhs}, {rhs})', ast.Sub: 'πg.ISub(πF, {lhs}, {rhs})', ast.BitXor: 'πg.IXor(πF, {lhs}, {rhs})', } def _assign_target(self, target, value): if isinstance(target, ast.Name): self.block.bind_var(self.writer, target.id, value) elif isinstance(target, ast.Attribute): with self.visit_expr(target.value) as obj: self.writer.write_checked_call1( 'πg.SetAttr(πF, {}, {}, {})', obj.expr, self.block.root.intern(target.attr), value) elif isinstance(target, ast.Subscript): with self.visit_expr(target.value) as mapping,\ self.visit_expr(target.slice) as index: self.writer.write_checked_call1('πg.SetItem(πF, {}, {}, {})', mapping.expr, index.expr, value) else: msg = 'assignment target not yet implemented: ' + type(target).__name__ raise util.ParseError(target, msg) def _build_assign_target(self, target, assigns): if isinstance(target, (ast.Tuple, ast.List)): children = [] for elt in target.elts: children.append(self._build_assign_target(elt, assigns)) tmpl = 'πg.TieTarget{Children: []πg.TieTarget{$children}}' return string.Template(tmpl).substitute(children=', '.join(children)) temp = self.block.alloc_temp() assigns.append((target, temp)) tmpl = 'πg.TieTarget{Target: &$temp}' return string.Template(tmpl).substitute(temp=temp.name) def _import_and_bind(self, imp): """Generates code that imports a module and binds it to a variable. Args: imp: Import object representing an import of the form "import x.y.z" or "from x.y import z". Expects only a single binding. """ # Acquire handles to the Code objects in each Go package and call # ImportModule to initialize all modules. with self.block.alloc_temp() as mod, \ self.block.alloc_temp('[]*πg.Object') as mod_slice: self.writer.write_checked_call2( mod_slice, 'πg.ImportModule(πF, {})', util.go_str(imp.name)) # Bind the imported modules or members to variables in the current scope. for binding in imp.bindings: if binding.bind_type == imputil.Import.MODULE: self.writer.write('{} = {}[{}]'.format( mod.name, mod_slice.expr, binding.value)) self.block.bind_var(self.writer, binding.alias, mod.expr) else: self.writer.write('{} = {}[{}]'.format( mod.name, mod_slice.expr, imp.name.count('.'))) # Binding a member of the imported module. with self.block.alloc_temp() as member: self.writer.write_checked_call2( member, 'πg.GetAttr(πF, {}, {}, nil)', mod.expr, self.block.root.intern(binding.value)) self.block.bind_var(self.writer, binding.alias, member.expr) def _tie_target(self, target, value): if isinstance(target, ast.Name): self._assign_target(target, value) return assigns = [] self.writer.write_checked_call1( 'πg.Tie(πF, {}, {})', self._build_assign_target(target, assigns), value) for t, temp in assigns: self._assign_target(t, temp.expr) self.block.free_temp(temp) def _visit_each(self, nodes): for node in nodes: self.visit(node) def _visit_loop(self, testfunc, node): start_label = self.block.genlabel(is_checkpoint=True) else_label = self.block.genlabel(is_checkpoint=True) end_label = self.block.genlabel() with self.block.alloc_temp('bool') as breakvar: self.block.push_loop(breakvar) self.writer.write('πF.PushCheckpoint({})'.format(else_label)) self.writer.write('{} = false'.format(breakvar.name)) self.writer.write_label(start_label) self.writer.write_tmpl(textwrap.dedent("""\ if πE != nil || πR != nil { \tcontinue } if $breakvar { \tπF.PopCheckpoint() \tgoto Label$end_label }"""), breakvar=breakvar.expr, end_label=end_label) with self.block.alloc_temp('bool') as testvar: testfunc(testvar) self.writer.write_tmpl(textwrap.dedent("""\ if πE != nil || !$testvar { \tcontinue } πF.PushCheckpoint($start_label)\ """), testvar=testvar.name, start_label=start_label) self._visit_each(node.body) self.writer.write('continue') # End the loop so that break applies to an outer loop if present. self.block.pop_loop() self.writer.write_label(else_label) self.writer.write(textwrap.dedent("""\ if πE != nil || πR != nil { \tcontinue }""")) if node.orelse: self._visit_each(node.orelse) self.writer.write_label(end_label) def _write_except_block(self, label, exc, except_node): self._write_py_context(except_node.lineno) self.writer.write_label(label) if except_node.name: self.block.bind_var(self.writer, except_node.name.id, '{}.ToObject()'.format(exc)) self._visit_each(except_node.body) self.writer.write('πF.RestoreExc(nil, nil)') def _write_except_dispatcher(self, exc, tb, handlers): """Outputs a Go code that jumps to the appropriate except handler. Args: exc: Go variable holding the current exception. tb: Go variable holding the current exception's traceback. handlers: A list of ast.ExceptHandler nodes. Returns: A list of Go labels indexes corresponding to the exception handlers. Raises: ParseError: Except handlers are in an invalid order. """ handler_labels = [] for i, except_node in enumerate(handlers): handler_labels.append(self.block.genlabel()) if except_node.type: with self.visit_expr(except_node.type) as type_,\ self.block.alloc_temp('bool') as is_inst: self.writer.write_checked_call2( is_inst, 'πg.IsInstance(πF, {}.ToObject(), {})', exc, type_.expr) self.writer.write_tmpl(textwrap.dedent("""\ if $is_inst { \tgoto Label$label }"""), is_inst=is_inst.expr, label=handler_labels[-1]) else: # This is a bare except. It should be the last handler. if i != len(handlers) - 1: msg = "default 'except:' must be last" raise util.ParseError(except_node, msg) self.writer.write('goto Label{}'.format(handler_labels[-1])) if handlers[-1].type: # There's no bare except, so the fallback is to re-raise. self.writer.write( 'πE = πF.Raise({}.ToObject(), nil, {}.ToObject())'.format(exc, tb)) self.writer.write('continue') return handler_labels def _write_py_context(self, lineno): if lineno: line = self.block.root.buffer.source_line(lineno).strip() self.writer.write('// line {}: {}'.format(lineno, line)) self.writer.write('πF.SetLineno({})'.format(lineno)) ================================================ FILE: compiler/stmt_test.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for StatementVisitor.""" from __future__ import unicode_literals import re import subprocess import textwrap import unittest from grumpy.compiler import block from grumpy.compiler import imputil from grumpy.compiler import shard_test from grumpy.compiler import stmt from grumpy.compiler import util from grumpy import pythonparser from grumpy.pythonparser import ast class StatementVisitorTest(unittest.TestCase): def testAssertNoMsg(self): self.assertEqual((0, 'AssertionError()\n'), _GrumpRun(textwrap.dedent("""\ try: assert False except AssertionError as e: print repr(e)"""))) def testAssertMsg(self): want = (0, "AssertionError('foo',)\n") self.assertEqual(want, _GrumpRun(textwrap.dedent("""\ try: assert False, 'foo' except AssertionError as e: print repr(e)"""))) def testBareAssert(self): # Assertion errors at the top level of a block should raise: # https://github.com/google/grumpy/issues/18 want = (0, 'ok\n') self.assertEqual(want, _GrumpRun(textwrap.dedent("""\ def foo(): assert False try: foo() except AssertionError: print 'ok' else: print 'bad'"""))) def testAssignAttribute(self): self.assertEqual((0, '123\n'), _GrumpRun(textwrap.dedent("""\ e = Exception() e.foo = 123 print e.foo"""))) def testAssignName(self): self.assertEqual((0, 'bar\n'), _GrumpRun(textwrap.dedent("""\ foo = 'bar' print foo"""))) def testAssignMultiple(self): self.assertEqual((0, 'baz baz\n'), _GrumpRun(textwrap.dedent("""\ foo = bar = 'baz' print foo, bar"""))) def testAssignSubscript(self): self.assertEqual((0, "{'bar': None}\n"), _GrumpRun(textwrap.dedent("""\ foo = {} foo['bar'] = None print foo"""))) def testAssignTuple(self): self.assertEqual((0, 'a b\n'), _GrumpRun(textwrap.dedent("""\ baz = ('a', 'b') foo, bar = baz print foo, bar"""))) def testAugAssign(self): self.assertEqual((0, '42\n'), _GrumpRun(textwrap.dedent("""\ foo = 41 foo += 1 print foo"""))) def testAugAssignBitAnd(self): self.assertEqual((0, '3\n'), _GrumpRun(textwrap.dedent("""\ foo = 7 foo &= 3 print foo"""))) def testAugAssignPow(self): self.assertEqual((0, '64\n'), _GrumpRun(textwrap.dedent("""\ foo = 8 foo **= 2 print foo"""))) def testClassDef(self): self.assertEqual((0, "\n"), _GrumpRun(textwrap.dedent("""\ class Foo(object): pass print type(Foo)"""))) def testClassDefWithVar(self): self.assertEqual((0, 'abc\n'), _GrumpRun(textwrap.dedent("""\ class Foo(object): bar = 'abc' print Foo.bar"""))) def testDeleteAttribute(self): self.assertEqual((0, 'False\n'), _GrumpRun(textwrap.dedent("""\ class Foo(object): bar = 42 del Foo.bar print hasattr(Foo, 'bar')"""))) def testDeleteClassLocal(self): self.assertEqual((0, 'False\n'), _GrumpRun(textwrap.dedent("""\ class Foo(object): bar = 'baz' del bar print hasattr(Foo, 'bar')"""))) def testDeleteGlobal(self): self.assertEqual((0, 'False\n'), _GrumpRun(textwrap.dedent("""\ foo = 42 del foo print 'foo' in globals()"""))) def testDeleteLocal(self): self.assertEqual((0, 'ok\n'), _GrumpRun(textwrap.dedent("""\ def foo(): bar = 123 del bar try: print bar raise AssertionError except UnboundLocalError: print 'ok' foo()"""))) def testDeleteNonexistentLocal(self): self.assertRaisesRegexp( util.ParseError, 'cannot delete nonexistent local', _ParseAndVisit, 'def foo():\n del bar') def testDeleteSubscript(self): self.assertEqual((0, '{}\n'), _GrumpRun(textwrap.dedent("""\ foo = {'bar': 'baz'} del foo['bar'] print foo"""))) def testExprCall(self): self.assertEqual((0, 'bar\n'), _GrumpRun(textwrap.dedent("""\ def foo(): print 'bar' foo()"""))) def testExprNameGlobal(self): self.assertEqual((0, ''), _GrumpRun(textwrap.dedent("""\ foo = 42 foo"""))) def testExprNameLocal(self): self.assertEqual((0, ''), _GrumpRun(textwrap.dedent("""\ foo = 42 def bar(): foo bar()"""))) def testFor(self): self.assertEqual((0, '1\n2\n3\n'), _GrumpRun(textwrap.dedent("""\ for i in (1, 2, 3): print i"""))) def testForBreak(self): self.assertEqual((0, '1\n'), _GrumpRun(textwrap.dedent("""\ for i in (1, 2, 3): print i break"""))) def testForContinue(self): self.assertEqual((0, '1\n2\n3\n'), _GrumpRun(textwrap.dedent("""\ for i in (1, 2, 3): print i continue raise AssertionError"""))) def testForElse(self): self.assertEqual((0, 'foo\nbar\n'), _GrumpRun(textwrap.dedent("""\ for i in (1,): print 'foo' else: print 'bar'"""))) def testForElseBreakNotNested(self): self.assertRaisesRegexp( util.ParseError, "'continue' not in loop", _ParseAndVisit, 'for i in (1,):\n pass\nelse:\n continue') def testForElseContinueNotNested(self): self.assertRaisesRegexp( util.ParseError, "'continue' not in loop", _ParseAndVisit, 'for i in (1,):\n pass\nelse:\n continue') def testFunctionDecorator(self): self.assertEqual((0, 'foo\n'), _GrumpRun(textwrap.dedent("""\ def bold(fn): return lambda: '' + fn() + '' @bold def foo(): return 'foo' print foo()"""))) def testFunctionDecoratorWithArg(self): self.assertEqual((0, 'foo\n'), _GrumpRun(textwrap.dedent("""\ def tag(name): def bold(fn): return lambda: '' + fn() + '' return bold @tag('red') def foo(): return 'foo' print foo()"""))) def testFunctionDef(self): self.assertEqual((0, 'bar baz\n'), _GrumpRun(textwrap.dedent("""\ def foo(a, b): print a, b foo('bar', 'baz')"""))) def testFunctionDefGenerator(self): self.assertEqual((0, "['foo', 'bar']\n"), _GrumpRun(textwrap.dedent("""\ def gen(): yield 'foo' yield 'bar' print list(gen())"""))) def testFunctionDefGeneratorReturnValue(self): self.assertRaisesRegexp( util.ParseError, 'returning a value in a generator function', _ParseAndVisit, 'def foo():\n yield 1\n return 2') def testFunctionDefLocal(self): self.assertEqual((0, 'baz\n'), _GrumpRun(textwrap.dedent("""\ def foo(): def bar(): print 'baz' bar() foo()"""))) def testIf(self): self.assertEqual((0, 'foo\n'), _GrumpRun(textwrap.dedent("""\ if 123: print 'foo' if '': print 'bar'"""))) def testIfElif(self): self.assertEqual((0, 'foo\nbar\n'), _GrumpRun(textwrap.dedent("""\ if True: print 'foo' elif False: print 'bar' if False: print 'foo' elif True: print 'bar'"""))) def testIfElse(self): self.assertEqual((0, 'foo\nbar\n'), _GrumpRun(textwrap.dedent("""\ if True: print 'foo' else: print 'bar' if False: print 'foo' else: print 'bar'"""))) def testImport(self): self.assertEqual((0, "\n"), _GrumpRun(textwrap.dedent("""\ import sys print type(sys.modules)"""))) def testImportFutureLateRaises(self): regexp = 'from __future__ imports must occur at the beginning of the file' self.assertRaisesRegexp(util.ImportError, regexp, _ParseAndVisit, 'foo = bar\nfrom __future__ import print_function') def testFutureUnicodeLiterals(self): want = "u'foo'\n" self.assertEqual((0, want), _GrumpRun(textwrap.dedent("""\ from __future__ import unicode_literals print repr('foo')"""))) def testImportMember(self): self.assertEqual((0, "\n"), _GrumpRun(textwrap.dedent("""\ from sys import modules print type(modules)"""))) def testImportConflictingPackage(self): self.assertEqual((0, ''), _GrumpRun(textwrap.dedent("""\ import time from "__go__/time" import Now"""))) def testImportNative(self): self.assertEqual((0, '1 1000000000\n'), _GrumpRun(textwrap.dedent("""\ from "__go__/time" import Nanosecond, Second print Nanosecond, Second"""))) def testImportGrumpy(self): self.assertEqual((0, ''), _GrumpRun(textwrap.dedent("""\ from "__go__/grumpy" import Assert Assert(__frame__(), True, 'bad')"""))) def testImportNativeType(self): self.assertEqual((0, "\n"), _GrumpRun(textwrap.dedent("""\ from "__go__/time" import Duration print Duration"""))) def testImportWildcardMemberRaises(self): regexp = 'wildcard member import is not implemented' self.assertRaisesRegexp(util.ImportError, regexp, _ParseAndVisit, 'from foo import *') self.assertRaisesRegexp(util.ImportError, regexp, _ParseAndVisit, 'from "__go__/foo" import *') def testPrintStatement(self): self.assertEqual((0, 'abc 123\nfoo bar\n'), _GrumpRun(textwrap.dedent("""\ print 'abc', print '123' print 'foo', 'bar'"""))) def testPrintFunction(self): want = "abc\n123\nabc 123\nabcx123\nabc 123 " self.assertEqual((0, want), _GrumpRun(textwrap.dedent("""\ "module docstring is ok to proceed __future__" from __future__ import print_function print('abc') print(123) print('abc', 123) print('abc', 123, sep='x') print('abc', 123, end=' ')"""))) def testRaiseExitStatus(self): self.assertEqual(1, _GrumpRun('raise Exception')[0]) def testRaiseInstance(self): self.assertEqual((0, 'foo\n'), _GrumpRun(textwrap.dedent("""\ try: raise RuntimeError('foo') print 'bad' except RuntimeError as e: print e"""))) def testRaiseTypeAndArg(self): self.assertEqual((0, 'foo\n'), _GrumpRun(textwrap.dedent("""\ try: raise KeyError('foo') print 'bad' except KeyError as e: print e"""))) def testRaiseAgain(self): self.assertEqual((0, 'foo\n'), _GrumpRun(textwrap.dedent("""\ try: try: raise AssertionError('foo') except AssertionError: raise except Exception as e: print e"""))) def testRaiseTraceback(self): self.assertEqual((0, ''), _GrumpRun(textwrap.dedent("""\ import sys try: try: raise Exception except: e, _, tb = sys.exc_info() raise e, None, tb except: e2, _, tb2 = sys.exc_info() assert e is e2 assert tb is tb2"""))) def testReturn(self): self.assertEqual((0, 'bar\n'), _GrumpRun(textwrap.dedent("""\ def foo(): return 'bar' print foo()"""))) def testTryBareExcept(self): self.assertEqual((0, ''), _GrumpRun(textwrap.dedent("""\ try: raise AssertionError except: pass"""))) def testTryElse(self): self.assertEqual((0, 'foo baz\n'), _GrumpRun(textwrap.dedent("""\ try: print 'foo', except: print 'bar' else: print 'baz'"""))) def testTryMultipleExcept(self): self.assertEqual((0, 'bar\n'), _GrumpRun(textwrap.dedent("""\ try: raise AssertionError except RuntimeError: print 'foo' except AssertionError: print 'bar' except: print 'baz'"""))) def testTryFinally(self): result = _GrumpRun(textwrap.dedent("""\ try: print 'foo', finally: print 'bar' try: print 'foo', raise Exception finally: print 'bar'""")) self.assertEqual(1, result[0]) self.assertIn('foo bar\nfoo bar\n', result[1]) self.assertIn('Exception\n', result[1]) def testWhile(self): self.assertEqual((0, '2\n1\n'), _GrumpRun(textwrap.dedent("""\ i = 2 while i: print i i -= 1"""))) def testWhileElse(self): self.assertEqual((0, 'bar\n'), _GrumpRun(textwrap.dedent("""\ while False: print 'foo' else: print 'bar'"""))) def testWith(self): self.assertEqual((0, 'enter\n1\nexit\nenter\n2\nexit\n3\n'), _GrumpRun(textwrap.dedent("""\ class ContextManager(object): def __enter__(self): print "enter" def __exit__(self, exc_type, value, traceback): print "exit" a = ContextManager() with a: print 1 try: with a: print 2 raise RuntimeError except RuntimeError: print 3 """))) def testWithAs(self): self.assertEqual((0, '1 2 3\n'), _GrumpRun(textwrap.dedent("""\ class ContextManager(object): def __enter__(self): return (1, (2, 3)) def __exit__(self, *args): pass with ContextManager() as [x, (y, z)]: print x, y, z """))) def testWriteExceptDispatcherBareExcept(self): visitor = stmt.StatementVisitor(_MakeModuleBlock()) handlers = [ast.ExceptHandler(type=ast.Name(id='foo')), ast.ExceptHandler(type=None)] self.assertEqual(visitor._write_except_dispatcher( # pylint: disable=protected-access 'exc', 'tb', handlers), [1, 2]) expected = re.compile(r'ResolveGlobal\(.*foo.*\bIsInstance\(.*' r'goto Label1.*goto Label2', re.DOTALL) self.assertRegexpMatches(visitor.writer.getvalue(), expected) def testWriteExceptDispatcherBareExceptionNotLast(self): visitor = stmt.StatementVisitor(_MakeModuleBlock()) handlers = [ast.ExceptHandler(type=None), ast.ExceptHandler(type=ast.Name(id='foo'))] self.assertRaisesRegexp(util.ParseError, r"default 'except:' must be last", visitor._write_except_dispatcher, # pylint: disable=protected-access 'exc', 'tb', handlers) def testWriteExceptDispatcherMultipleExcept(self): visitor = stmt.StatementVisitor(_MakeModuleBlock()) handlers = [ast.ExceptHandler(type=ast.Name(id='foo')), ast.ExceptHandler(type=ast.Name(id='bar'))] self.assertEqual(visitor._write_except_dispatcher( # pylint: disable=protected-access 'exc', 'tb', handlers), [1, 2]) expected = re.compile( r'ResolveGlobal\(.*foo.*\bif .*\bIsInstance\(.*\{.*goto Label1.*' r'ResolveGlobal\(.*bar.*\bif .*\bIsInstance\(.*\{.*goto Label2.*' r'\bRaise\(exc\.ToObject\(\), nil, tb\.ToObject\(\)\)', re.DOTALL) self.assertRegexpMatches(visitor.writer.getvalue(), expected) def _MakeModuleBlock(): return block.ModuleBlock(None, '__main__', '', '', imputil.FutureFeatures()) def _ParseAndVisit(source): mod = pythonparser.parse(source) _, future_features = imputil.parse_future_features(mod) importer = imputil.Importer(None, 'foo', 'foo.py', False) b = block.ModuleBlock(importer, '__main__', '', source, future_features) visitor = stmt.StatementVisitor(b) visitor.visit(mod) return visitor def _GrumpRun(cmd): p = subprocess.Popen(['grumprun'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, _ = p.communicate(cmd) return p.returncode, out if __name__ == '__main__': shard_test.main() ================================================ FILE: compiler/util.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for generating Go code.""" from __future__ import unicode_literals import codecs import contextlib import cStringIO import string import StringIO import textwrap _SIMPLE_CHARS = set(string.digits + string.letters + string.punctuation + " ") _ESCAPES = {'\t': r'\t', '\r': r'\r', '\n': r'\n', '"': r'\"', '\\': r'\\'} # This is the max length of a direct allocation tuple supported by the runtime. # This should match the number of specializations found in tuple.go. MAX_DIRECT_TUPLE = 6 class CompileError(Exception): def __init__(self, node, msg): if hasattr(node, 'lineno'): msg = 'line {}: {}'.format(node.lineno, msg) super(CompileError, self).__init__(msg) class ParseError(CompileError): pass class ImportError(CompileError): # pylint: disable=redefined-builtin pass class LateFutureError(ImportError): def __init__(self, node): msg = 'from __future__ imports must occur at the beginning of the file' super(LateFutureError, self).__init__(node, msg) class Writer(object): """Utility class for writing blocks of Go code to a file-like object.""" def __init__(self, out=None): self.out = codecs.getwriter('utf8')(out or cStringIO.StringIO()) self.indent_level = 0 def getvalue(self): return self.out.getvalue().decode('utf8') @contextlib.contextmanager def indent_block(self, n=1): """A context manager that indents by n on entry and dedents on exit.""" self.indent(n) yield self.dedent(n) def write(self, output): for line in output.split('\n'): if line: self.out.write(''.join(('\t' * self.indent_level, line, '\n'))) def write_block(self, block_, body): """Outputs the boilerplate necessary for code blocks like functions. Args: block_: The Block object representing the code block. body: String containing Go code making up the body of the code block. """ self.write('for ; πF.State() >= 0; πF.PopCheckpoint() {') with self.indent_block(): self.write('switch πF.State() {') self.write('case 0:') for checkpoint in block_.checkpoints: self.write_tmpl('case $state: goto Label$state', state=checkpoint) self.write('default: panic("unexpected function state")') self.write('}') # Assume that body is aligned with goto labels. with self.indent_block(-1): self.write(body) self.write('}') def write_label(self, label): with self.indent_block(-1): self.write('Label{}:'.format(label)) def write_py_context(self, lineno, line): self.write_tmpl('// line $lineno: $line', lineno=lineno, line=line) def write_tmpl(self, tmpl, **kwargs): self.write(string.Template(tmpl).substitute(kwargs)) def write_checked_call2(self, result, call, *args, **kwargs): return self.write_tmpl(textwrap.dedent("""\ if $result, πE = $call; πE != nil { \tcontinue }"""), result=result.name, call=call.format(*args, **kwargs)) def write_checked_call1(self, call, *args, **kwargs): return self.write_tmpl(textwrap.dedent("""\ if πE = $call; πE != nil { \tcontinue }"""), call=call.format(*args, **kwargs)) def write_temp_decls(self, block_): all_temps = block_.free_temps | block_.used_temps for temp in sorted(all_temps, key=lambda t: t.name): self.write('var {0} {1}\n_ = {0}'.format(temp.name, temp.type_)) def indent(self, n=1): self.indent_level += n def dedent(self, n=1): self.indent_level -= n def go_str(value): """Returns value as a valid Go string literal.""" io = StringIO.StringIO() io.write('"') for c in value: if c in _ESCAPES: io.write(_ESCAPES[c]) elif c in _SIMPLE_CHARS: io.write(c) else: io.write(r'\x{:02x}'.format(ord(c))) io.write('"') return io.getvalue() def adjust_local_name(name): """Returns a Go identifier for the given Python variable name.""" return 'µ' + name ================================================ FILE: compiler/util_test.py ================================================ # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests Writer and other utils.""" from __future__ import unicode_literals import unittest from grumpy.compiler import block from grumpy.compiler import imputil from grumpy.compiler import util class WriterTest(unittest.TestCase): def testIndentBlock(self): writer = util.Writer() writer.write('foo') with writer.indent_block(n=2): writer.write('bar') writer.write('baz') self.assertEqual(writer.getvalue(), 'foo\n\t\tbar\nbaz\n') def testWriteBlock(self): writer = util.Writer() mod_block = block.ModuleBlock(None, '__main__', '', '', imputil.FutureFeatures()) writer.write_block(mod_block, 'BODY') output = writer.getvalue() dispatch = 'switch πF.State() {\n\tcase 0:\n\tdefault: panic' self.assertIn(dispatch, output) def testWriteMultiline(self): writer = util.Writer() writer.indent(2) writer.write('foo\nbar\nbaz\n') self.assertEqual(writer.getvalue(), '\t\tfoo\n\t\tbar\n\t\tbaz\n') def testWritePyContext(self): writer = util.Writer() writer.write_py_context(12, 'print "foo"') self.assertEqual(writer.getvalue(), '// line 12: print "foo"\n') def testWriteSkipBlankLine(self): writer = util.Writer() writer.write('foo\n\nbar') self.assertEqual(writer.getvalue(), 'foo\nbar\n') def testWriteTmpl(self): writer = util.Writer() writer.write_tmpl('$foo, $bar\n$baz', foo=1, bar=2, baz=3) self.assertEqual(writer.getvalue(), '1, 2\n3\n') def testIndent(self): writer = util.Writer() writer.indent(2) writer.write('foo') self.assertEqual(writer.getvalue(), '\t\tfoo\n') def testDedent(self): writer = util.Writer() writer.indent(4) writer.dedent(3) writer.write('foo') self.assertEqual(writer.getvalue(), '\tfoo\n') if __name__ == '__main__': unittest.main() ================================================ FILE: lib/README.md ================================================ This directory contains Grumpy Python standard library code. It is only expected to run under Grumpy, not CPython. ================================================ FILE: lib/__builtin__.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Built-in Python identifiers.""" # pylint: disable=invalid-name from '__go__/grumpy' import Builtins for k, v in Builtins.iteritems(): globals()[k] = v ================================================ FILE: lib/_random.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Generate pseudo random numbers. Should not be used for security purposes.""" from '__go__/math/rand' import Uint32, Seed from '__go__/math' import Pow from '__go__/time' import Now BPF = 53 # Number of bits in a float RECIP_BPF = Pow(2, -BPF) # TODO: The random byte generator currently uses math.rand.Uint32 to generate # 4 bytes at a time. We should use math.rand.Read to generate the correct # number of bytes needed. This can be changed once there is a way to # allocate the needed []byte for Read from python and cast it to a list of # integers once it is filled. def _gorandom(nbytes): byte_arr = [] while len(byte_arr) < nbytes: i = Uint32() byte_arr.append(i & 0xff) byte_arr.append(i >> 8 & 0xff) byte_arr.append(i >> 16 & 0xff) byte_arr.append(i >> 24 & 0xff) byte_arr = byte_arr[0:nbytes] return byte_arr # This is a slow replacement for int.bit_length. # We should stop using this if it is implemented. def _int_bit_length(n): bits = 0 while n: # 1 bit steps n = n / 2 bits += 1 return bits # Replacement for int.from_bytes (big endian) def _int_from_bytes(bytes): i = 0 n = len(bytes) - 1 while n >= 0: i += bytes[n] << (8 * n) n -= 1 return i class GrumpyRandom(object): """Random generator replacement for Grumpy. Alternate random number generator using golangs math.rand as a replacement for the CPython implementation. """ def random(self): """Get the next random number in the range [0.0, 1.0).""" return (_int_from_bytes(_gorandom(7)) >> 3) * RECIP_BPF def getrandbits(self, k): """getrandbits(k) -> x. Generates an int with k random bits.""" if k <= 0: raise ValueError('number of bits must be greater than zero') if k != int(k): raise TypeError('number of bits should be an integer') numbytes = (k + 7) // 8 # bits / 8 and rounded up x = _int_from_bytes(_gorandom(numbytes)) return x >> (numbytes * 8 - k) # trim excess bits def seed(self, a=None): """Seed the golang.math.rand generator.""" if a is None: a = Now().UnixNano() Seed(a) def _randbelow(self, n): """Return a random int in the range [0,n).""" # TODO # change once int.bit_length is implemented. # k = n.bit_length() k = _int_bit_length(n) r = self.getrandbits(k) while r >= n: r = self.getrandbits(k) return r def getstate(self, *args, **kwargs): raise NotImplementedError('Entropy source does not have state.') def setstate(self, *args, **kwargs): raise NotImplementedError('Entropy source does not have state.') def jumpahead(self, *args, **kwargs): raise NotImplementedError('Entropy source does not have state.') ================================================ FILE: lib/_syscall.py ================================================ # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from '__go__/syscall' import EINTR def invoke(func, *args): while True: result = func(*args) if isinstance(result, tuple): err = result[-1] result = result[:-1] else: err = result result = () if err: if err == EINTR: continue raise OSError(err.Error()) return result ================================================ FILE: lib/errno.py ================================================ EINVAL = 22 ================================================ FILE: lib/exceptions.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Built-in exception classes.""" from '__go__/grumpy' import ExceptionTypes g = globals() for t in ExceptionTypes: g[t.__name__] = t ================================================ FILE: lib/itertools.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utilities for iterating over containers.""" import _collections import sys class chain(object): def from_iterable(cls, iterables): for it in iterables: for element in it: yield element from_iterable = classmethod(from_iterable) def __init__(self, *iterables): if not iterables: self.iterables = iter([[]]) else: self.iterables = iter(iterables) self.curriter = iter(next(self.iterables)) def __iter__(self): return self def next(self): flag = True while flag: try: ret = next(self.curriter) flag = False except StopIteration: self.curriter = iter(next(self.iterables)) return ret def compress(data, selectors): return (d for d,s in izip(data, selectors) if s) def count(start=0, step=1): n = start while True: yield n n += step def cycle(iterable): saved = [] for element in iterable: yield element saved.append(element) while saved: for element in saved: yield element def dropwhile(predicate, iterable): iterable = iter(iterable) for x in iterable: if not predicate(x): yield x break for x in iterable: yield x class groupby(object): # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D def __init__(self, iterable, key=None): if key is None: key = lambda x: x self.keyfunc = key self.it = iter(iterable) self.tgtkey = self.currkey = self.currvalue = object() def __iter__(self): return self def next(self): while self.currkey == self.tgtkey: self.currvalue = next(self.it) # Exit on StopIteration self.currkey = self.keyfunc(self.currvalue) self.tgtkey = self.currkey return (self.currkey, self._grouper(self.tgtkey)) def _grouper(self, tgtkey): while self.currkey == tgtkey: yield self.currvalue self.currvalue = next(self.it) # Exit on StopIteration self.currkey = self.keyfunc(self.currvalue) def ifilter(predicate, iterable): if predicate is None: predicate = bool for x in iterable: if predicate(x): yield x def ifilterfalse(predicate, iterable): if predicate is None: predicate = bool for x in iterable: if not predicate(x): yield x def imap(function, *iterables): iterables = map(iter, iterables) while True: args = [next(it) for it in iterables] if function is None: yield tuple(args) else: yield function(*args) def islice(iterable, *args): s = slice(*args) it = iter(xrange(s.start or 0, s.stop or sys.maxint, s.step or 1)) nexti = next(it) for i, element in enumerate(iterable): if i == nexti: yield element nexti = next(it) def izip(*iterables): iterators = map(iter, iterables) while iterators: yield tuple(map(next, iterators)) class ZipExhausted(Exception): pass def izip_longest(*args, **kwds): # izip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D- fillvalue = kwds.get('fillvalue') counter = [len(args) - 1] def sentinel(): if not counter[0]: raise ZipExhausted counter[0] -= 1 yield fillvalue fillers = repeat(fillvalue) iterators = [chain(it, sentinel(), fillers) for it in args] try: while iterators: yield tuple(map(next, iterators)) except ZipExhausted: pass def product(*args, **kwds): # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 pools = map(tuple, args) * kwds.get('repeat', 1) result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] for prod in result: yield tuple(prod) def permutations(iterable, r=None): pool = tuple(iterable) n = len(pool) r = n if r is None else r for indices in product(range(n), repeat=r): if len(set(indices)) == r: yield tuple(pool[i] for i in indices) def combinations(iterable, r): pool = tuple(iterable) n = len(pool) for indices in permutations(range(n), r): if sorted(indices) == list(indices): yield tuple(pool[i] for i in indices) def combinations_with_replacement(iterable, r): pool = tuple(iterable) n = len(pool) for indices in product(range(n), repeat=r): if sorted(indices) == list(indices): yield tuple(pool[i] for i in indices) def repeat(object, times=None): if times is None: while True: yield object else: for i in xrange(times): yield object def starmap(function, iterable): for args in iterable: yield function(*args) def takewhile(predicate, iterable): for x in iterable: if predicate(x): yield x else: break def tee(iterable, n=2): it = iter(iterable) deques = [_collections.deque() for i in range(n)] def gen(mydeque): while True: if not mydeque: newval = next(it) for d in deques: d.append(newval) yield mydeque.popleft() return tuple(gen(d) for d in deques) ================================================ FILE: lib/itertools_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import itertools import weetest def TestCycle(): want = [] got = [] for x in itertools.cycle(()): got.append(x) assert got == want, 'empty cycle yields no elements' arg = (0, 1, 2) want = (0, 1, 2) * 10 got = [] limit = 10 * len(arg) counter = 0 for x in itertools.cycle((0, 1, 2)): got.append(x) counter += 1 if counter == limit: break assert tuple(got) == want, 'tuple(cycle%s) == %s, want %s' % (arg, tuple(got), want) def TestDropwhile(): r = range(10) cases = [ ((lambda x: x < 5, r), (5, 6, 7, 8, 9)), ((lambda x: True, r), ()), ((lambda x: False, r), tuple(r)), ] for args, want in cases: got = tuple(itertools.dropwhile(*args)) assert got == want, 'tuple(dropwhile%s) == %s, want %s' % (args, got, want) def TestChain(): r = range(10) cases = [ ([r], tuple(r)), ([r, r], tuple(r) + tuple(r)), ([], ()) ] for args, want in cases: got = tuple(itertools.chain(*args)) assert got == want, 'tuple(chain%s) == %s, want %s' % (args, got, want) def TestFromIterable(): r = range(10) cases = [ ([r], tuple(r)), ([r, r], tuple(r) + tuple(r)), ([], ()) ] for args, want in cases: got = tuple(itertools.chain.from_iterable(args)) assert got == want, 'tuple(from_iterable%s) == %s, want %s' % (args, got, want) def TestIFilter(): r = range(10) cases = [ ((lambda x: x < 5, r), (0, 1, 2, 3, 4)), ((lambda x: False, r), ()), ((lambda x: True, r), tuple(r)), ((None, r), (1, 2, 3, 4, 5, 6, 7, 8, 9)) ] for args, want in cases: got = tuple(itertools.ifilter(*args)) assert got == want, 'tuple(ifilter%s) == %s, want %s' % (args, got, want) def TestIFilterFalse(): r = range(10) cases = [ ((lambda x: x < 5, r), (5, 6, 7, 8, 9)), ((lambda x: False, r), tuple(r)), ((lambda x: True, r), ()), ((None, r), (0,)) ] for args, want in cases: got = tuple(itertools.ifilterfalse(*args)) assert got == want, 'tuple(ifilterfalse%s) == %s, want %s' % (args, got, want) def TestISlice(): r = range(10) cases = [ ((r, 5), (0, 1, 2, 3, 4)), ((r, 25, 30), ()), ((r, 1, None, 3), (1, 4, 7)), ] for args, want in cases: got = tuple(itertools.islice(*args)) assert got == want, 'tuple(islice%s) == %s, want %s' % (args, got, want) def TestIZipLongest(): cases = [ (('abc', range(6)), (('a', 0), ('b', 1), ('c', 2), (None, 3), (None, 4), (None, 5))), ((range(6), 'abc'), ((0, 'a'), (1, 'b'), (2, 'c'), (3, None), (4, None), (5, None))), (([1, None, 3], 'ab', range(1)), ((1, 'a', 0), (None, 'b', None), (3, None, None))), ] for args, want in cases: got = tuple(itertools.izip_longest(*args)) assert got == want, 'tuple(izip_longest%s) == %s, want %s' % (args, got, want) def TestProduct(): cases = [ (([1, 2], ['a', 'b']), ((1, 'a'), (1, 'b'), (2, 'a'), (2, 'b'))), (([1], ['a', 'b']), ((1, 'a'), (1, 'b'))), (([],), ()), ] for args, want in cases: got = tuple(itertools.product(*args)) assert got == want, 'tuple(product%s) == %s, want %s' % (args, got, want) def TestPermutations(): cases = [ (('AB',), (('A', 'B'), ('B', 'A'))), (('ABC', 2), (('A', 'B'), ('A', 'C'), ('B', 'A'), ('B', 'C'), ('C', 'A'), ('C', 'B'))), ((range(3),), ((0, 1, 2), (0, 2, 1), (1, 0, 2), (1, 2, 0), (2, 0, 1), (2, 1, 0))), (([],), ((),)), (([], 0), ((),)), ((range(3), 4), ()), ] for args, want in cases: got = tuple(itertools.permutations(*args)) assert got == want, 'tuple(permutations%s) == %s, want %s' % (args, got, want) def TestCombinations(): cases = [ ((range(4), 3), ((0, 1, 2), (0, 1, 3), (0, 2, 3), (1, 2, 3))), ] for args, want in cases: got = tuple(itertools.combinations(*args)) assert got == want, 'tuple(combinations%s) == %s, want %s' % (args, got, want) def TestCombinationsWithReplacement(): cases = [ (([-12], 2), (((-12, -12),))), (('AB', 3), (('A', 'A', 'A'), ('A', 'A', 'B'), ('A', 'B', 'B'), ('B', 'B', 'B'))), (([], 2), ()), (([], 0), ((),)) ] for args, want in cases: got = tuple(itertools.combinations_with_replacement(*args)) assert got == want, 'tuple(combinations_with_replacement%s) == %s, want %s' % (args, got, want) def TestGroupBy(): cases = [ (([1, 2, 2, 3, 3, 3, 4, 4, 4, 4],), [(1, [1]), (2, [2, 2]), (3, [3, 3, 3]), (4, [4, 4, 4, 4])]), ((['aa', 'ab', 'abc', 'bcd', 'abcde'], len), [(2, ['aa', 'ab']), (3, ['abc', 'bcd']), (5, ['abcde'])]), ] for args, want in cases: got = [(k, list(v)) for k, v in itertools.groupby(*args)] assert got == want, 'groupby %s == %s, want %s' % (args, got, want) def TestTakewhile(): r = range(10) cases = [ ((lambda x: x % 2 == 0, r), (0,)), ((lambda x: True, r), tuple(r)), ((lambda x: False, r), ()) ] for args, want in cases: got = tuple(itertools.takewhile(*args)) assert got == want, 'tuple(takewhile%s) == %s, want %s' % (args, got, want) if __name__ == '__main__': weetest.RunTests() ================================================ FILE: lib/math.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from '__go__/math' import (Pi, E, Ceil, Copysign, Abs, Floor, Mod, Frexp, IsInf, IsNaN, Exp2, Modf, Trunc, Exp, Expm1, Log, Log1p, Log10, Pow, Sqrt, Acos, Asin, Atan, Atan2, Hypot, Sin, Cos, Tan, Acosh, Asinh, Atanh, Sinh, Cosh, Tanh, Erf, Erfc, Gamma, Lgamma) # pylint: disable=g-multiple-import # Constants pi = Pi e = E # Number-theoretic and representation functions def ceil(x): return Ceil(float(x)) def copysign(x, y): return Copysign(float(x), float(y)) def fabs(x): return Abs(float(x)) def factorial(x): try: xi = int(x) except TypeError: xi = None try: xf = float(x) except TypeError: xf = None if xi is None: xi = int(xf) if xi != xf: raise ValueError("factorial() only accepts integral values") elif xf is None and xi is None: raise TypeError("an integer is required") elif xf is None: pass elif xf != xi: raise ValueError("factorial() only accepts integral values") x = xi if x < 0: raise ValueError("factorial() not defined for negative values") acc = 1 for value in range(2, x+1): acc *= value return acc def floor(x): return Floor(float(x)) def fmod(x): return Mod(float(x)) def frexp(x): return Frexp(float(x)) # TODO: Implement fsum() # def fsum(x): # pass def isinf(x): return IsInf(float(x), 0) def isnan(x): return IsNaN(float(x)) def ldexp(x, i): return float(x) * Exp2(float(i)) def modf(x): # Modf returns (int, frac), but python should return (frac, int). a, b = Modf(float(x)) return b, a def trunc(x): return Trunc(float(x)) # Power and logarithmic functions def exp(x): return Exp(float(x)) def expm1(x): return Expm1(float(x)) def log(x, b=None): if b is None: return Log(float(x)) # NOTE: We can try and catch more special cases to delegate to specific # Go functions or maybe there is a function that does this and I missed it. return Log(float(x)) / Log(float(b)) def log1p(x): return Log1p(float(x)) def log10(x): return Log10(float(x)) def pow(x, y): return Pow(float(x), float(y)) def sqrt(x): return Sqrt(float(x)) # Trigonometric functions def acos(x): return Acos(float(x)) def asin(x): return Asin(float(x)) def atan(x): return Atan(float(x)) def atan2(y, x): return Atan2(float(y), float(x)) def cos(x): return Cos(float(x)) def hypot(x, y): return Hypot(float(x), float(y)) def sin(x): return Sin(float(x)) def tan(x): return Tan(float(x)) # Angular conversion def degrees(x): return (float(x) * 180) / pi def radians(x): return (float(x) * pi) / 180 # Hyperbolic functions def acosh(x): return Acosh(float(x)) def asinh(x): return Asinh(float(x)) def atanh(x): return Atanh(float(x)) def cosh(x): return Cosh(float(x)) def sinh(x): return Sinh(float(x)) def tanh(x): return Tanh(float(x)) # Special functions def erf(x): return Erf(float(x)) def erfc(x): return Erfc(float(x)) def gamma(x): return Gamma(float(x)) def lgamma(x): return Lgamma(float(x)) ================================================ FILE: lib/math_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import math import weetest # Tests exist for all functions which have logic in the math.py module, instead # of simply calling the go equivalent. def TestFactorial(): assert math.factorial(0) == 1 assert math.factorial(1) == 1 assert math.factorial(2) == 2 assert math.factorial(3) == 6 assert math.factorial(4) == 24 assert math.factorial(5) == 120 def TestFactorialError(): try: math.factorial(-1) except ValueError: pass else: raise AssertionError try: math.factorial(0.5) except ValueError: pass else: raise AssertionError def TestLdexp(): assert math.ldexp(1,1) == 2 assert math.ldexp(1,2) == 4 assert math.ldexp(1.5,1) == 3 assert math.ldexp(1.5,2) == 6 def TestLog(): assert math.log(math.e) == 1 assert math.log(2,2) == 1 assert math.log(10,10) == 1 assert math.log(100,10) == 2 def TestRadians(): assert math.radians(180) == math.pi assert math.radians(360) == 2 * math.pi def TestDegrees(): assert math.degrees(math.pi) == 180 assert math.degrees(2 * math.pi) == 360 if __name__ == '__main__': weetest.RunTests() ================================================ FILE: lib/os/__init__.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Miscellaneous operating system interfaces.""" # pylint: disable=g-multiple-import from '__go__/io/ioutil' import ReadDir from '__go__/os' import (Chdir, Chmod, Environ, Getpid as getpid, Getwd, Pipe, ProcAttr, Remove, StartProcess, Stat, Stdout, Stdin, Stderr, Mkdir) from '__go__/path/filepath' import Separator from '__go__/grumpy' import (NewFileFromFD, StartThread, ToNative) from '__go__/reflect' import MakeSlice from '__go__/runtime' import GOOS from '__go__/syscall' import (Close, SYS_FCNTL, Syscall, F_GETFD, Wait4, WaitStatus, WNOHANG) from '__go__/sync' import WaitGroup from '__go__/time' import Second import _syscall from os import path import stat as stat_module import sys sep = chr(Separator) error = OSError # pylint: disable=invalid-name curdir = '.' name = 'posix' environ = {} for var in Environ(): k, v = var.split('=', 1) environ[k] = v def mkdir(path, mode=0o777): err = Mkdir(path, mode) if err: raise OSError(err.Error()) def chdir(path): err = Chdir(path) if err: raise OSError(err.Error()) def chmod(filepath, mode): # TODO: Support mode flags other than perms. err = Chmod(filepath, stat(filepath).st_mode & ~0o777 | mode & 0o777) if err: raise OSError(err.Error()) def close(fd): err = Close(fd) if err: raise OSError(err.Error()) def fdopen(fd, mode='r'): # pylint: disable=unused-argument # Ensure this is a valid file descriptor to match CPython behavior. _, _, err = Syscall(SYS_FCNTL, fd, F_GETFD, 0) if err: raise OSError(err.Error()) return NewFileFromFD(fd, None) def listdir(p): files, err = ReadDir(p) if err: raise OSError(err.Error()) return [x.Name() for x in files] def getcwd(): dir, err = Getwd() if err: raise OSError(err.Error()) return dir class _Popen(object): def __init__(self, command, mode): self.mode = mode self.result = None self.r, self.w, err = Pipe() if err: raise OSError(err.Error()) attr = ProcAttr.new() # Create a slice using a reflect.Type returned by ToNative. # TODO: There should be a cleaner way to create slices in Python. files_type = ToNative(__frame__(), attr.Files).Type() files = MakeSlice(files_type, 3, 3).Interface() if self.mode == 'r': fd = self.r.Fd() files[0], files[1], files[2] = Stdin, self.w, Stderr elif self.mode == 'w': fd = self.w.Fd() files[0], files[1], files[2] = self.r, Stdout, Stderr else: raise ValueError('invalid popen mode: %r', self.mode) attr.Files = files # TODO: There should be a cleaner way to create slices in Python. args_type = ToNative(__frame__(), StartProcess).Type().In(1) args = MakeSlice(args_type, 3, 3).Interface() shell = environ['SHELL'] args[0] = shell args[1] = '-c' args[2] = command self.proc, err = StartProcess(shell, args, attr) if err: raise OSError(err.Error()) self.wg = WaitGroup.new() self.wg.Add(1) StartThread(self._thread_func) self.file = NewFileFromFD(fd, self.close) def _thread_func(self): self.result = self.proc.Wait() if self.mode == 'r': self.w.Close() self.wg.Done() def close(self, _): if self.mode == 'w': self.w.Close() self.wg.Wait() state, err = self.result if err: raise OSError(err.Error()) return state.Sys() def popen(command, mode='r'): return _Popen(command, mode).file def remove(filepath): if stat_module.S_ISDIR(stat(filepath).st_mode): raise OSError('Operation not permitted: ' + filepath) err = Remove(filepath) if err: raise OSError(err.Error()) def rmdir(filepath): if not stat_module.S_ISDIR(stat(filepath).st_mode): raise OSError('Operation not permitted: ' + filepath) err = Remove(filepath) if err: raise OSError(err.Error()) class StatResult(object): def __init__(self, info): self._info = info def st_mode(self): # TODO: This is an incomplete mode flag. It should include S_IFDIR, etc. return self._info.Mode() # TODO: Make this a decorator once they're implemented. st_mode = property(st_mode) def st_mtime(self): return float(self._info.ModTime().UnixNano()) / Second # TODO: Make this a decorator once they're implemented. st_mtime = property(st_mtime) def st_size(self): return self._info.Size() # TODO: Make this a decorator once they're implemented. st_size = property(st_size) def stat(filepath): info, err = Stat(filepath) if err: raise OSError(err.Error()) return StatResult(info) unlink = remove def waitpid(pid, options): status = WaitStatus.new() _syscall.invoke(Wait4, pid, status, options, None) return pid, _encode_wait_result(status) def _encode_wait_result(status): return status.Signal() | (status.ExitStatus() << 8) ================================================ FILE: lib/os/path.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """"Utilities for manipulating and inspecting OS paths.""" from '__go__/os' import Stat from '__go__/path/filepath' import Abs, Base, Clean, Dir as dirname, IsAbs as isabs, Join, Split # pylint: disable=g-multiple-import,unused-import def abspath(path): result, err = Abs(path) if err: raise OSError(err.Error()) if isinstance(path, unicode): # Grumpy compiler encoded the string into utf-8, so the result can be # decoded using utf-8. return unicode(result, 'utf-8') return result def basename(path): return '' if path.endswith('/') else Base(path) def exists(path): _, err = Stat(path) return err is None def isdir(path): info, err = Stat(path) if info and err is None: return info.Mode().IsDir() return False def isfile(path): info, err = Stat(path) if info and err is None: return info.Mode().IsRegular() return False # NOTE(compatibility): This method uses Go's filepath.Join() method which # implicitly normalizes the resulting path (pruning extra /, .., etc.) The usual # CPython behavior is to leave all the cruft. This deviation is reasonable # because a) result paths will point to the same files and b) one cannot assume # much about the results of join anyway since it's platform dependent. def join(*paths): if not paths: raise TypeError('join() takes at least 1 argument (0 given)') parts = [] for p in paths: if isabs(p): parts = [p] else: parts.append(p) result = Join(*parts) if result and not paths[-1]: result += '/' return result def normpath(path): result = Clean(path) if isinstance(path, unicode): return unicode(result, 'utf-8') return result def split(path): head, tail = Split(path) if len(head) > 1 and head[-1] == '/': head = head[:-1] return (head, tail) ================================================ FILE: lib/os/path_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=g-import-not-at-top import os import os.path path = os.path import weetest import tempfile def _AssertEqual(a, b): assert a == b assert type(a) is type(b) def TestAbspath(): _AssertEqual(path.abspath('/a/b/c'), '/a/b/c') _AssertEqual(path.abspath(u'/a/b/c'), u'/a/b/c') _AssertEqual(path.abspath('/a/b/c/'), '/a/b/c') _AssertEqual(path.abspath(u'/a/b/c/'), u'/a/b/c') _AssertEqual(path.abspath('a/b/c'), path.normpath(os.getcwd() + '/a/b/c')) def TestBasename(): assert path.basename('/a/b/c') == 'c' assert path.basename('/a/b/c/') == '' def TestDirname(): assert path.dirname('/a/b/c') == '/a/b' assert path.dirname('/a/b/c/') == '/a/b/c' def TestExists(): _, file_path = tempfile.mkstemp() dir_path = tempfile.mkdtemp() try: assert path.exists(file_path) assert path.exists(dir_path) assert not path.exists('path/does/not/exist') finally: os.remove(file_path) os.rmdir(dir_path) def TestIsAbs(): assert path.isabs('/abc') assert not path.isabs('abc/123') def TestIsDir(): _, file_path = tempfile.mkstemp() dir_path = tempfile.mkdtemp() try: assert not path.isdir(file_path) assert path.isdir(dir_path) assert not path.isdir('path/does/not/exist') finally: os.remove(file_path) os.rmdir(dir_path) def TestIsFile(): _, file_path = tempfile.mkstemp() dir_path = tempfile.mkdtemp() try: assert path.isfile(file_path) assert not path.isfile(dir_path) assert not path.isfile('path/does/not/exist') finally: os.remove(file_path) os.rmdir(dir_path) def TestJoin(): assert path.join('') == '' assert path.join('', '') == '' assert path.join('abc') == 'abc' assert path.join('abc', '') == 'abc/' assert path.join('abc', '', '') == 'abc/' assert path.join('abc', '', '123') == 'abc/123' assert path.normpath(path.join('abc', '.', '123')) == 'abc/123' assert path.normpath(path.join('abc', '..', '123')) == '123' assert path.join('/abc', '123') == '/abc/123' assert path.join('abc', '/123') == '/123' assert path.join('abc/', '123') == 'abc/123' assert path.join('abc', 'x/y/z') == 'abc/x/y/z' assert path.join('abc', 'x', 'y', 'z') == 'abc/x/y/z' def TestNormPath(): _AssertEqual(path.normpath('abc/'), 'abc') _AssertEqual(path.normpath('/a//b'), '/a/b') _AssertEqual(path.normpath('abc/../123'), '123') _AssertEqual(path.normpath('../abc/123'), '../abc/123') _AssertEqual(path.normpath('x/y/./z'), 'x/y/z') _AssertEqual(path.normpath(u'abc/'), u'abc') _AssertEqual(path.normpath(u'/a//b'), u'/a/b') _AssertEqual(path.normpath(u'abc/../123'), u'123') _AssertEqual(path.normpath(u'../abc/123'), u'../abc/123') _AssertEqual(path.normpath(u'x/y/./z'), u'x/y/z') def TestSplit(): assert path.split('a/b') == ('a', 'b') assert path.split('a/b/') == ('a/b', '') assert path.split('a/') == ('a', '') assert path.split('a') == ('', 'a') assert path.split('/') == ('/', '') assert path.split('/a/./b') == ('/a/.', 'b') if __name__ == '__main__': weetest.RunTests() ================================================ FILE: lib/os_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import stat import time import tempfile import weetest def TestChdirAndGetCwd(): path = os.getcwd() os.chdir('.') assert os.getcwd() == path tempdir = tempfile.mkdtemp() try: os.chdir(tempdir) assert tempdir in os.getcwd() finally: os.chdir(path) os.rmdir(tempdir) assert os.getcwd() == path def TestChmod(): fd, path = tempfile.mkstemp() os.close(fd) os.chmod(path, 0o644) mode = os.stat(path).st_mode & 0o777 os.remove(path) assert mode == 0o644 def TestChmodOSError(): tempdir = tempfile.mkdtemp() try: os.chmod(tempdir + '/DoesNotExist', 0o644) except OSError: pass else: raise AssertionError def TestClose(): fd, _ = tempfile.mkstemp() os.close(fd) try: os.fdopen(fd) except OSError: pass else: raise AssertionError def TestCloseOSError(): fd, _ = tempfile.mkstemp() os.close(fd) try: os.close(fd) except OSError: pass else: raise AssertionError def TestEnviron(): assert 'HOME' in os.environ def TestFDOpen(): fd, path = tempfile.mkstemp() f = os.fdopen(fd, 'w') f.write('foobar') f.close() f = open(path) contents = f.read() f.close() assert contents == 'foobar', contents def TestFDOpenOSError(): fd, _ = tempfile.mkstemp() os.close(fd) try: os.fdopen(fd) except OSError: pass else: raise AssertionError def TestMkdir(): path = 'foobarqux' try: os.stat(path) except OSError: pass else: raise AssertionError try: os.mkdir(path) assert stat.S_ISDIR(os.stat(path).st_mode) except OSError: raise AssertionError finally: os.rmdir(path) def TestPopenRead(): f = os.popen('qux') assert f.close() == 32512 f = os.popen('echo hello') try: assert f.read() == 'hello\n' finally: assert f.close() == 0 def TestPopenWrite(): # TODO: We should verify the output but there's no good way to swap out stdout # at the moment. f = os.popen('cat', 'w') f.write('popen write\n') f.close() def TestRemove(): fd, path = tempfile.mkstemp() os.close(fd) os.stat(path) os.remove(path) try: os.stat(path) except OSError: pass else: raise AssertionError def TestRemoveNoExist(): path = tempfile.mkdtemp() try: os.remove(path + '/nonexistent') except OSError: pass else: raise AssertionError finally: os.rmdir(path) def TestRemoveDir(): path = tempfile.mkdtemp() try: os.remove(path) except OSError: pass else: raise AssertionError finally: os.rmdir(path) def TestRmDir(): path = tempfile.mkdtemp() assert stat.S_ISDIR(os.stat(path).st_mode) os.rmdir(path) try: os.stat(path) except OSError: pass else: raise AssertionError def TestRmDirNoExist(): path = tempfile.mkdtemp() try: os.rmdir(path + '/nonexistent') except OSError: pass else: raise AssertionError finally: os.rmdir(path) def TestRmDirFile(): fd, path = tempfile.mkstemp() os.close(fd) try: os.rmdir(path) except OSError: pass else: raise AssertionError finally: os.remove(path) def TestStatFile(): t = time.time() fd, path = tempfile.mkstemp() os.close(fd) st = os.stat(path) os.remove(path) assert not stat.S_ISDIR(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o600 # System time and mtime may have different precision so give 10 sec leeway. assert st.st_mtime + 10 > t assert st.st_size == 0 def TestStatDir(): path = tempfile.mkdtemp() mode = os.stat(path).st_mode os.rmdir(path) assert stat.S_ISDIR(mode) assert stat.S_IMODE(mode) == 0o700 def TestStatNoExist(): path = tempfile.mkdtemp() try: os.stat(path + '/nonexistent') except OSError: pass else: raise AssertionError finally: os.rmdir(path) def TestWaitPid(): try: pid, status = os.waitpid(-1, os.WNOHANG) except OSError as e: assert 'no child processes' in str(e).lower() if __name__ == '__main__': weetest.RunTests() ================================================ FILE: lib/random_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import _random import random import weetest def TestGrumpyRandom(): assert len(_random._gorandom(5)) == 5 assert _random._int_bit_length(0) == 0 assert _random._int_bit_length(1) == 1 assert _random._int_bit_length(8) == 4 assert _random._int_bit_length(256) == 9 assert _random._int_from_bytes([1, 0, 0, 0]) == 1 assert _random._int_from_bytes([0, 0, 0, 0]) == 0 assert _random._int_from_bytes([255, 255, 0, 0]) == 65535 assert _random._int_from_bytes([0, 0, 0, 1]) == 16777216 r = _random.GrumpyRandom() assert 0.0 <= r.random() < 1.0 assert 0 <= r.getrandbits(1) <= 1 assert 0 <= r.getrandbits(2) <= 3 assert 0 <= r.getrandbits(8) <= 255 assert 0 <= r._randbelow(1) < 1 assert 0 <= r._randbelow(3) < 3 assert 0 <= r._randbelow(1000) < 1000 def TestSeed(): random.seed() try: random.seed("wrongtype") except TypeError: pass else: raise AssertionError("TypeError not raised") def TestRandom(): a = random.random() b = random.random() c = random.random() assert isinstance(a, float) assert 0.0 <= a < 1.0 assert not a == b == c def TestRandomUniform(): for _ in range(10): a = random.uniform(0, 1000) assert isinstance(a, float) assert 0 <= a <= 1000 def TestRandomInt(): for _ in range(10): a = random.randint(0, 1000000) assert isinstance(a, int) assert 0 <= a <= 1000000 b = random.randint(1, 1) assert b == 1 try: c = random.randint(0.1, 3) except ValueError: pass else: raise AssertionError("ValueError not raised") try: d = random.randint(4, 3) except ValueError: pass else: raise AssertionError("ValueError not raised") def TestRandomChoice(): seq = [i*2 for i in range(5)] for i in range(10): item = random.choice(seq) item_idx = item/2 assert seq[item_idx] == item try: random.choice([]) except IndexError: pass else: raise AssertionError("IndexError not raised") if __name__ == '__main__': weetest.RunTests() ================================================ FILE: lib/select_.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from '__go__/syscall' import ( FD_SETSIZE as _FD_SETSIZE, Select as _Select, FdSet as _FdSet, Timeval as _Timeval ) import _syscall import math class error(Exception): pass def select(rlist, wlist, xlist, timeout=None): rlist_norm = _normalize_fd_list(rlist) wlist_norm = _normalize_fd_list(wlist) xlist_norm = _normalize_fd_list(xlist) all_fds = rlist_norm + wlist_norm + xlist_norm if not all_fds: nfd = 0 else: nfd = max(all_fds) + 1 rfds = _make_fdset(rlist_norm) wfds = _make_fdset(wlist_norm) xfds = _make_fdset(xlist_norm) if timeout is None: timeval = None else: timeval = _Timeval.new() frac, integer = math.modf(timeout) timeval.Sec = int(integer) timeval.Usec = int(frac * 1000000.0) _syscall.invoke(_Select, nfd, rfds, wfds, xfds, timeval) return ([rlist[i] for i, fd in enumerate(rlist_norm) if _fdset_isset(fd, rfds)], [wlist[i] for i, fd in enumerate(wlist_norm) if _fdset_isset(fd, wfds)], [xlist[i] for i, fd in enumerate(xlist_norm) if _fdset_isset(fd, xfds)]) def _fdset_set(fd, fds): idx = fd / (_FD_SETSIZE / len(fds.Bits)) % len(fds.Bits) pos = fd % (_FD_SETSIZE / len(fds.Bits)) fds.Bits[idx] |= 1 << pos def _fdset_isset(fd, fds): idx = fd / (_FD_SETSIZE / len(fds.Bits)) % len(fds.Bits) pos = fd % (_FD_SETSIZE / len(fds.Bits)) return bool(fds.Bits[idx] & (1 << pos)) def _make_fdset(fd_list): fds = _FdSet.new() for fd in fd_list: _fdset_set(fd, fds) return fds def _normalize_fd_list(fds): result = [] # Python permits mutating the select fds list during fileno calls so we can't # just use simple iteration over the list. See test_select_mutated in # test_select.py i = 0 while i < len(fds): fd = fds[i] if hasattr(fd, 'fileno'): fd = fd.fileno() result.append(fd) i += 1 return result ================================================ FILE: lib/stat.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Interpreting stat() results.""" # pylint: disable=g-multiple-import from '__go__/os' import ModeDir, ModePerm def S_ISDIR(mode): # pylint: disable=invalid-name return mode & ModeDir != 0 def S_IMODE(mode): # pylint: disable=invalid-name return mode & ModePerm ================================================ FILE: lib/sys.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """System-specific parameters and functions.""" from '__go__/os' import Args from '__go__/grumpy' import SysModules, MaxInt, Stdin as stdin, Stdout as stdout, Stderr as stderr # pylint: disable=g-multiple-import from '__go__/runtime' import (GOOS as platform, Version) from '__go__/unicode' import MaxRune argv = [] for arg in Args: argv.append(arg) goversion = Version() maxint = MaxInt maxsize = maxint maxunicode = MaxRune modules = SysModules py3kwarning = False warnoptions = [] # TODO: Support actual byteorder byteorder = 'little' version = '2.7.13' class _Flags(object): """Container class for sys.flags.""" debug = 0 py3k_warning = 0 division_warning = 0 division_new = 0 inspect = 0 interactive = 0 optimize = 0 dont_write_bytecode = 0 no_user_site = 0 no_site = 0 ignore_environment = 0 tabcheck = 0 verbose = 0 unicode = 0 bytes_warning = 0 hash_randomization = 0 flags = _Flags() def exc_clear(): __frame__().__exc_clear__() def exc_info(): e, tb = __frame__().__exc_info__() # pylint: disable=undefined-variable t = None if e: t = type(e) return t, e, tb def exit(code=None): # pylint: disable=redefined-builtin raise SystemExit(code) def _getframe(depth=0): f = __frame__() while depth > 0 and f is not None: f = f.f_back depth -= 1 if f is None: raise ValueError('call stack is not deep enough') return f ================================================ FILE: lib/sys_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=bare-except import sys import types import weetest def TestArgv(): assert sys.argv def TestMaxInt(): assert sys.maxint > 2000000000 def TestSysModules(): assert sys.modules['sys'] is not None def TestExcClear(): try: raise RuntimeError except: assert all(sys.exc_info()), sys.exc_info() sys.exc_clear() assert not any(sys.exc_info()) else: assert False def TestExcInfoNoException(): assert sys.exc_info() == (None, None, None) def TestExcInfoWithException(): try: raise RuntimeError except: t, e, tb = sys.exc_info() else: assert False assert t is RuntimeError assert isinstance(e, t) assert isinstance(tb, types.TracebackType) def TestExitEmpty(): try: sys.exit() except SystemExit as e: assert e.code == None, e.code # pylint: disable=g-equals-none except: assert False def TestExitCode(): try: sys.exit(42) except SystemExit as e: assert e.code == 42, e.code except: assert False def TestExitInvalidArgs(): try: sys.exit(1, 2, 3) except TypeError as e: assert str(e) == 'exit() takes 1 arguments (3 given)', str(e) except: assert False def TestGetFrame(): try: sys._getframe(42, 42) except TypeError: pass else: assert False try: sys._getframe(2000000000) except ValueError: pass else: assert False assert sys._getframe().f_code.co_name == '_getframe' assert sys._getframe(1).f_code.co_name == 'TestGetFrame' if __name__ == '__main__': # This call will incidentally test sys.exit(). weetest.RunTests() ================================================ FILE: lib/tempfile.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Generate temporary files and directories.""" # pylint: disable=g-multiple-import from '__go__/io/ioutil' import TempDir, TempFile from '__go__/syscall' import Dup # pylint: disable=redefined-builtin def mkdtemp(suffix='', prefix='tmp', dir=None): if dir is None: dir = '' # TODO: Make suffix actually follow the rest of the filename. path, err = TempDir(dir, prefix + '-' + suffix) if err: raise OSError(err.Error()) return path def mkstemp(suffix='', prefix='tmp', dir=None, text=False): if text: raise NotImplementedError if dir is None: dir = '' # TODO: Make suffix actually follow the rest of the filename. f, err = TempFile(dir, prefix + '-' + suffix) if err: raise OSError(err.Error()) try: fd, err = Dup(f.Fd()) if err: raise OSError(err.Error()) return fd, f.Name() finally: f.Close() ================================================ FILE: lib/tempfile_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import stat import tempfile import weetest def TestMkdTemp(): path = tempfile.mkdtemp() mode = os.stat(path).st_mode os.rmdir(path) assert stat.S_ISDIR(mode), mode assert stat.S_IMODE(mode) == 0o700, mode def TestMkdTempDir(): tempdir = tempfile.mkdtemp() path = tempfile.mkdtemp(dir=tempdir) os.rmdir(path) os.rmdir(tempdir) assert path.startswith(tempdir) def TestMkdTempOSError(): tempdir = tempfile.mkdtemp() os.chmod(tempdir, 0o500) try: tempfile.mkdtemp(dir=tempdir) except OSError: pass else: raise AssertionError os.rmdir(tempdir) def TestMkdTempPrefixSuffix(): path = tempfile.mkdtemp(prefix='foo', suffix='bar') os.rmdir(path) assert 'foo' in path assert 'bar' in path # TODO: assert path.endswith('bar') def TestMksTemp(): fd, path = tempfile.mkstemp() f = os.fdopen(fd, 'w') f.write('foobar') f.close() f = open(path) contents = f.read() f.close() os.remove(path) assert contents == 'foobar', contents def TestMksTempDir(): tempdir = tempfile.mkdtemp() fd, path = tempfile.mkstemp(dir=tempdir) os.close(fd) os.remove(path) os.rmdir(tempdir) assert path.startswith(tempdir) def TestMksTempOSError(): tempdir = tempfile.mkdtemp() os.chmod(tempdir, 0o500) try: tempfile.mkstemp(dir=tempdir) except OSError: pass else: raise AssertionError os.rmdir(tempdir) def TestMksTempPerms(): fd, path = tempfile.mkstemp() os.close(fd) mode = os.stat(path).st_mode os.remove(path) assert stat.S_IMODE(mode) == 0o600, mode def TestMksTempPrefixSuffix(): fd, path = tempfile.mkstemp(prefix='foo', suffix='bar') os.close(fd) os.remove(path) assert 'foo' in path assert 'bar' in path # TODO: assert path.endswith('bar') if __name__ == '__main__': weetest.RunTests() ================================================ FILE: lib/thread.py ================================================ from '__go__/grumpy' import NewTryableMutex, StartThread, ThreadCount class error(Exception): pass def get_ident(): f = __frame__() while f.f_back: f = f.f_back return id(f) class LockType(object): def __init__(self): self._mutex = NewTryableMutex() def acquire(self, waitflag=1): if waitflag: self._mutex.Lock() return True return self._mutex.TryLock() def release(self): self._mutex.Unlock() def __enter__(self): self.acquire() def __exit__(self, *args): self.release() def allocate_lock(): """Dummy implementation of thread.allocate_lock().""" return LockType() def start_new_thread(func, args, kwargs=None): if kwargs is None: kwargs = {} l = allocate_lock() ident = [] def thread_func(): ident.append(get_ident()) l.release() func(*args, **kwargs) l.acquire() StartThread(thread_func) l.acquire() return ident[0] def stack_size(n=0): if n: raise error('grumpy does not support setting stack size') return 0 def _count(): return ThreadCount ================================================ FILE: lib/time.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Time access and conversions.""" from '__go__/time' import Local, Now, Second, Sleep, Unix, Date, UTC # pylint: disable=g-multiple-import _strftime_directive_map = { '%': '%', 'a': 'Mon', 'A': 'Monday', 'b': 'Jan', 'B': 'January', 'c': NotImplemented, 'd': '02', 'H': '15', 'I': '03', 'j': NotImplemented, 'L': '.000', 'm': '01', 'M': '04', 'p': 'PM', 'S': '05', 'U': NotImplemented, 'W': NotImplemented, 'w': NotImplemented, 'X': NotImplemented, 'x': NotImplemented, 'y': '06', 'Y': '2006', 'Z': 'MST', 'z': '-0700', } class struct_time(tuple): #pylint: disable=invalid-name,missing-docstring def __init__(self, args): super(struct_time, self).__init__(tuple, args) self.tm_year = self[0] self.tm_mon = self[1] self.tm_mday = self[2] self.tm_hour = self[3] self.tm_min = self[4] self.tm_sec = self[5] self.tm_wday = self[6] self.tm_yday = self[7] self.tm_isdst = self[8] def __repr__(self): return ("time.struct_time(tm_year=%s, tm_mon=%s, tm_mday=%s, " "tm_hour=%s, tm_min=%s, tm_sec=%s, tm_wday=%s, " "tm_yday=%s, tm_isdst=%s)") % self def __str__(self): return repr(self) def gmtime(seconds=None): t = (Unix(seconds, 0) if seconds else Now()).UTC() return struct_time((t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), (t.Weekday() + 6) % 7, t.YearDay(), 0)) def localtime(seconds=None): t = (Unix(seconds, 0) if seconds else Now()).Local() return struct_time((t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), (t.Weekday() + 6) % 7, t.YearDay(), 0)) def mktime(t): return float(Date(t[0], t[1], t[2], t[3], t[4], t[5], 0, Local).Unix()) def sleep(secs): Sleep(secs * Second) def time(): return float(Now().UnixNano()) / Second def strftime(format, tt=None): # pylint: disable=missing-docstring,redefined-builtin t = Unix(int(mktime(tt)), 0) if tt else Now() ret = [] prev, n = 0, format.find('%', 0, -1) while n != -1: ret.append(format[prev:n]) next_ch = format[n + 1] c = _strftime_directive_map.get(next_ch) if c is NotImplemented: raise NotImplementedError('Code: %' + next_ch + ' not yet supported') if c: ret.append(t.Format(c)) else: ret.append(format[n:n+2]) n += 2 prev, n = n, format.find('%', n, -1) ret.append(format[prev:]) return ''.join(ret) # TODO: Calculate real value for daylight saving. daylight = 0 # TODO: Use local DST instead of ''. tzname = (Now().Zone()[0], '') ================================================ FILE: lib/time_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time assert time.time() > 1000000000 assert time.time() < 3000000000 time_struct = (1999, 9, 19, 0, 0, 0, 6, 262, 0) got = time.localtime(time.mktime(time_struct)) assert got == time_struct, got ================================================ FILE: lib/types_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import types from '__go__/grumpy' import (FunctionType, MethodType, ModuleType, StrType, # pylint: disable=g-multiple-import TracebackType, TypeType) # Verify a sample of all types as a sanity check. assert types.FunctionType is FunctionType assert types.MethodType is MethodType assert types.UnboundMethodType is MethodType assert types.ModuleType is ModuleType assert types.StringType is StrType assert types.TracebackType is TracebackType assert types.TypeType is TypeType ================================================ FILE: lib/weetest.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Minimal framework for writing tests and benchmarks. Follows a similar pattern to the Go testing package functionality. To implement and have test and benchmark functions run automatically, define Test/BenchmarkXyz() functions in your __main__ module and run them with weetest.RunTests/Benchmarks() For example: import weetest def BenchmarkOneThing(b): for i in xrange(b.N): OneThing() def BenchmarkAnotherThing(b): i = 0 while i < b.N: AnotherThing() i += 1 if __name__ == '__main__': weetest.RunBenchmarks() """ import os import sys import time import traceback # pylint: disable=invalid-name class _Benchmark(object): """Wraps and runs a single user defined benchmark function.""" def __init__(self, bench_func, target_duration): """Set up this benchmark to run bench_func to be run for target_duration.""" self.bench_func = bench_func self.target_duration = target_duration self.start_time = 0 self.N = 1 def Run(self): """Attempt to run this benchmark for the target duration.""" small_duration = 0.05 * self.target_duration self.N = 1 self._RunOnce() while self.duration < self.target_duration: if self.duration < small_duration: # Grow N very quickly when duration is small. N = self.N * 10 else: # Once duration is > 5% of target_duration we should have a good # estimate for how many iterations it will take to hit the target. Shoot # for 20% beyond that so we don't end up hovering just below the target. N = int(self.N * (1.2 * self.target_duration / self.duration)) if self.N == N: self.N += 1 else: self.N = N self._RunOnce() def _RunOnce(self): self.start_time = time.time() self.bench_func(self) self.duration = time.time() - self.start_time def ResetTimer(self): """Clears the current elapsed time to discount expensive setup steps.""" self.start_time = time.time() class _TestResult(object): """The outcome of running a particular benchmark function.""" def __init__(self, name): self.name = name self.status = 'not run' self.duration = 0 self.properties = {} def _RunOneBenchmark(name, test_func): """Runs a single benchmark and returns a _TestResult.""" b = _Benchmark(test_func, 1) result = _TestResult(name) print name, start_time = time.time() try: b.Run() except Exception as e: # pylint: disable=broad-except result.status = 'error' print 'ERROR' traceback.print_exc() else: result.status = 'passed' ops_per_sec = b.N / b.duration result.properties['ops_per_sec'] = ops_per_sec print 'PASSED', ops_per_sec finally: result.duration = time.time() - start_time return result def _RunOneTest(name, test_func): """Runs a single test function and returns a _TestResult.""" result = _TestResult(name) start_time = time.time() try: test_func() except AssertionError as e: result.status = 'failed' print name, 'FAILED' traceback.print_exc() except Exception as e: # pylint: disable=broad-except result.status = 'error' print name, 'ERROR' traceback.print_exc() else: result.status = 'passed' finally: result.duration = time.time() - start_time return result def _WriteXmlFile(filename, suite_duration, results): """Given a list of _BenchmarkResults, writes XML test results to filename.""" xml_file = open(filename, 'w') xml_file.write('\n' % (sys.argv[0], len(results), suite_duration)) for result in results: xml_file.write(' \n' % (result.name, result.duration)) if result.properties: xml_file.write(' \n') for name in result.properties: value = result.properties[name] if isinstance(value, float): formatted = '%f' % value else: formatted = str(value) xml_file.write(' \n' % (name, formatted)) xml_file.write(' \n') xml_file.write(' \n') xml_file.write('') xml_file.close() def _RunAll(test_prefix, runner): """Runs all functions in __main__ matching test_prefix using runner.""" target = os.environ.get('WEETEST_TARGET') exit_status = 0 mod = sys.modules['__main__'] results = [] suite_start_time = time.time() for name in dir(mod): if name.startswith(test_prefix) and (not target or name == target): result = runner(name, getattr(mod, name)) if result.status != 'passed': exit_status = 1 results.append(result) suite_duration = time.time() - suite_start_time if 'XML_OUTPUT_FILE' in os.environ: _WriteXmlFile(os.environ['XML_OUTPUT_FILE'], suite_duration, results) return exit_status def RunBenchmarks(): """Benchmarks all functions in __main__ with names like BenchmarkXyz().""" sys.exit(_RunAll('Benchmark', _RunOneBenchmark)) def RunTests(): """Runs all functions in __main__ with names like TestXyz().""" sys.exit(_RunAll('Test', _RunOneTest)) ================================================ FILE: lib/weetest_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import tempfile import time import weetest class _Timer(object): def __init__(self): self.t = 0 def Reset(self): self.t = 0 def time(self): # pylint: disable=invalid-name return self.t _timer = _Timer() # Stub out time for the whole test suite for convenience. time.time = _timer.time def TestBenchmark(): def BenchmarkFoo(b): i = 0 while i < b.N: i += 1.0 _timer.t += b.N b = weetest._Benchmark(BenchmarkFoo, 1000) _timer.Reset() b.Run() assert b.duration >= 1000, b.duration assert b.N >= 1000 def TestBenchmarkResetTimer(): def BenchmarkFoo(b): _timer.t += 10000 # Do an "expensive" initialization. b.ResetTimer() i = 0 while i < b.N: i += 1.0 _timer.t += b.N b = weetest._Benchmark(BenchmarkFoo, 1000) _timer.Reset() b.Run() assert b.duration < 2000, b.duration assert b.N < 2000, b.duration def TestRunOneBenchmark(): def BenchmarkFoo(b): i = 0 while i < b.N: i += 1.0 _timer.t += b.N _timer.Reset() result = weetest._RunOneBenchmark('BenchmarkFoo', BenchmarkFoo) assert result.status == 'passed' assert result.properties.get('ops_per_sec') == 1.0 assert result.duration == _timer.time() def TestRunOneBenchmarkError(): def BenchmarkFoo(unused_b): raise ValueError result = weetest._RunOneBenchmark('BenchmarkFoo', BenchmarkFoo) assert result.status == 'error' def TestRunOneTest(): def TestFoo(): # pylint: disable=undefined-loop-variable _timer.t += 100 if case[0]: raise case[0] # pylint: disable=raising-bad-type cases = [(None, 'passed'), (AssertionError, 'failed'), (ValueError, 'error')] for case in cases: _timer.Reset() result = weetest._RunOneTest('TestFoo', TestFoo) assert result.status == case[1] assert result.duration == 100 def TestWriteXmlFile(): result_with_properties = weetest._TestResult('foo') result_with_properties.properties['bar'] = 'baz' cases = [([], ['']), ([weetest._TestResult('foo')], [''])] for case in cases: fd, path = tempfile.mkstemp() os.close(fd) try: weetest._WriteXmlFile(path, 100, case[0]) f = open(path) contents = f.read() f.close() for want in case[1]: assert want in contents, contents finally: os.remove(path) def TestRunAll(): class Main(object): def __init__(self): self.run = {} def TestFoo(self): self.run['TestFoo'] = True def TestBar(self): self.run['TestBar'] = True def BenchmarkBaz(self, b): self.run['BenchmarkBaz'] = True _timer.t += b.N def BenchmarkQux(self, b): self.run['BenchmarkQux'] = True _timer.t += b.N fd, test_xml = tempfile.mkstemp() os.close(fd) fd, benchmark_xml = tempfile.mkstemp() os.close(fd) cases = [('Test', weetest._RunOneTest, None, {'TestFoo': True, 'TestBar': True}), ('Test', weetest._RunOneTest, test_xml, {'TestFoo': True, 'TestBar': True}), ('Benchmark', weetest._RunOneBenchmark, None, {'BenchmarkBaz': True, 'BenchmarkQux': True}), ('Benchmark', weetest._RunOneBenchmark, benchmark_xml, {'BenchmarkBaz': True, 'BenchmarkQux': True})] for prefix, runner, xml_path, want_run in cases: old_main = sys.modules['__main__'] sys.modules['__main__'] = main = Main() if xml_path: os.environ['XML_OUTPUT_FILE'] = xml_path try: weetest._RunAll(prefix, runner) finally: sys.modules['__main__'] = old_main if 'XML_OUTPUT_FILE' in os.environ: del os.environ['XML_OUTPUT_FILE'] assert main.run == want_run, main.run if xml_path: f = open(xml_path) xml = f.read() f.close() os.remove(xml_path) for name in want_run: assert '>4], hexTable[r&0x0F]} } if r < 0x10000 { return []byte{'\\', 'u', hexTable[r>>12], hexTable[r>>8&0x0F], hexTable[r>>4&0x0F], hexTable[r&0x0F]} } return []byte{'\\', 'U', hexTable[r>>28], hexTable[r>>24&0x0F], hexTable[r>>20&0x0F], hexTable[r>>16&0x0F], hexTable[r>>12&0x0F], hexTable[r>>8&0x0F], hexTable[r>>4&0x0F], hexTable[r&0x0F]} } ================================================ FILE: runtime/basestring_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestNormalizeEncoding(t *testing.T) { cases := []struct { encoding string want string }{ {"utf8", "utf8"}, {"UTF-16 ", "utf16"}, {" __Ascii__", "ascii"}, {"utf@#(%*#(*%16 ", "utf16"}, {"", ""}, } for _, cas := range cases { if got := normalizeEncoding(cas.encoding); got != cas.want { t.Errorf("normalizeEncoding(%q) = %q, want %q", cas.encoding, got, cas.want) } } } func BenchmarkEscapeRune(b *testing.B) { b.Run("low values", func(b *testing.B) { for i := 0; i < b.N; i++ { escapeRune(0x10) } }) b.Run("mid values", func(b *testing.B) { for i := 0; i < b.N; i++ { escapeRune(0x200) } }) b.Run("high values", func(b *testing.B) { for i := 0; i < b.N; i++ { escapeRune(0x20000) } }) } ================================================ FILE: runtime/bool.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" ) // GetBool returns True if v is true, False otherwise. func GetBool(v bool) *Int { if v { return True } return False } // BoolType is the object representing the Python 'bool' type. var BoolType = newSimpleType("bool", IntType) func boolNative(_ *Frame, o *Object) (reflect.Value, *BaseException) { return reflect.ValueOf(toIntUnsafe(o).Value() != 0), nil } func boolNew(f *Frame, _ *Type, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) if argc == 0 { return False.ToObject(), nil } if argc != 1 { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("bool() takes at most 1 argument (%d given)", argc)) } ret, raised := IsTrue(f, args[0]) if raised != nil { return nil, raised } return GetBool(ret).ToObject(), nil } func boolRepr(_ *Frame, o *Object) (*Object, *BaseException) { i := toIntUnsafe(o) if i.Value() != 0 { return trueStr, nil } return falseStr, nil } func initBoolType(map[string]*Object) { BoolType.flags &= ^(typeFlagInstantiable | typeFlagBasetype) BoolType.slots.Native = &nativeSlot{boolNative} BoolType.slots.New = &newSlot{boolNew} BoolType.slots.Repr = &unaryOpSlot{boolRepr} } var ( // True is the singleton bool object representing the Python 'True' // object. True = &Int{Object{typ: BoolType}, 1} // False is the singleton bool object representing the Python 'False' // object. False = &Int{Object{typ: BoolType}, 0} trueStr = NewStr("True").ToObject() falseStr = NewStr("False").ToObject() ) ================================================ FILE: runtime/bool_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestBoolCompare(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(true, true), want: compareAllResultEq}, {args: wrapArgs(true, false), want: compareAllResultGT}, {args: wrapArgs(true, 1), want: compareAllResultEq}, {args: wrapArgs(true, -1), want: compareAllResultGT}, {args: wrapArgs(false, 0), want: compareAllResultEq}, {args: wrapArgs(false, 1000), want: compareAllResultLT}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestBoolCreate(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(None), wantExc: mustCreateException(TypeErrorType, `'__new__' requires a 'type' object but received a "NoneType"`)}, {args: wrapArgs(BoolType), want: False.ToObject()}, {args: wrapArgs(BoolType, None), want: False.ToObject()}, {args: wrapArgs(BoolType, ""), want: False.ToObject()}, {args: wrapArgs(BoolType, true), want: True.ToObject()}, {args: wrapArgs(BoolType, newObject(ObjectType)), want: True.ToObject()}, {args: wrapArgs(ObjectType), wantExc: mustCreateException(TypeErrorType, "bool.__new__(object): object is not a subtype of bool")}, {args: wrapArgs(BoolType, "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "bool() takes at most 1 argument (2 given)")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(BoolType, "__new__", &cas); err != "" { t.Error(err) } } } func TestBoolStrRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(true), want: NewStr("True").ToObject()}, {args: wrapArgs(false), want: NewStr("False").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/builtin_types.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "math" "math/big" "strings" "unicode" ) var ( // Builtins contains all of the Python built-in identifiers. Builtins = NewDict() builtinStr = NewStr("__builtin__") // ExceptionTypes contains all builtin exception types. ExceptionTypes []*Type // EllipsisType is the object representing the Python 'ellipsis' type EllipsisType = newSimpleType("ellipsis", ObjectType) // Ellipsis is the singleton ellipsis object representing the Python // 'Ellipsis' object. Ellipsis = &Object{typ: EllipsisType} // NoneType is the object representing the Python 'NoneType' type. NoneType = newSimpleType("NoneType", ObjectType) // None is the singleton NoneType object representing the Python 'None' // object. None = &Object{typ: NoneType} // NotImplementedType is the object representing the Python // 'NotImplementedType' object. NotImplementedType = newSimpleType("NotImplementedType", ObjectType) // NotImplemented is the singleton NotImplementedType object // representing the Python 'NotImplemented' object. NotImplemented = newObject(NotImplementedType) unboundLocalType = newSimpleType("UnboundLocalType", ObjectType) // UnboundLocal is a singleton held by local variables in generated // code before they are bound. UnboundLocal = newObject(unboundLocalType) ) func ellipsisRepr(*Frame, *Object) (*Object, *BaseException) { return NewStr("Ellipsis").ToObject(), nil } func noneRepr(*Frame, *Object) (*Object, *BaseException) { return NewStr("None").ToObject(), nil } func notImplementedRepr(*Frame, *Object) (*Object, *BaseException) { return NewStr("NotImplemented").ToObject(), nil } func initEllipsisType(map[string]*Object) { EllipsisType.flags &= ^(typeFlagInstantiable | typeFlagBasetype) EllipsisType.slots.Repr = &unaryOpSlot{ellipsisRepr} } func initNoneType(map[string]*Object) { NoneType.flags &= ^(typeFlagInstantiable | typeFlagBasetype) NoneType.slots.Repr = &unaryOpSlot{noneRepr} } func initNotImplementedType(map[string]*Object) { NotImplementedType.flags &= ^(typeFlagInstantiable | typeFlagBasetype) NotImplementedType.slots.Repr = &unaryOpSlot{notImplementedRepr} } func initUnboundLocalType(map[string]*Object) { unboundLocalType.flags &= ^(typeFlagInstantiable | typeFlagBasetype) } type typeState int const ( typeStateNotReady typeState = iota typeStateInitializing typeStateReady ) type builtinTypeInit func(map[string]*Object) type builtinTypeInfo struct { state typeState init builtinTypeInit global bool } var builtinTypes = map[*Type]*builtinTypeInfo{ ArithmeticErrorType: {global: true}, AssertionErrorType: {global: true}, AttributeErrorType: {global: true}, BaseExceptionType: {init: initBaseExceptionType, global: true}, BaseStringType: {init: initBaseStringType, global: true}, BoolType: {init: initBoolType, global: true}, ByteArrayType: {init: initByteArrayType, global: true}, BytesWarningType: {global: true}, CodeType: {}, ComplexType: {init: initComplexType, global: true}, ClassMethodType: {init: initClassMethodType, global: true}, DeprecationWarningType: {global: true}, dictItemIteratorType: {init: initDictItemIteratorType}, dictKeyIteratorType: {init: initDictKeyIteratorType}, dictValueIteratorType: {init: initDictValueIteratorType}, DictType: {init: initDictType, global: true}, EllipsisType: {init: initEllipsisType, global: true}, enumerateType: {init: initEnumerateType, global: true}, EnvironmentErrorType: {global: true}, EOFErrorType: {global: true}, ExceptionType: {global: true}, FileType: {init: initFileType, global: true}, FloatType: {init: initFloatType, global: true}, FrameType: {init: initFrameType}, FrozenSetType: {init: initFrozenSetType, global: true}, FunctionType: {init: initFunctionType}, FutureWarningType: {global: true}, GeneratorType: {init: initGeneratorType}, ImportErrorType: {global: true}, ImportWarningType: {global: true}, IndexErrorType: {global: true}, IntType: {init: initIntType, global: true}, IOErrorType: {global: true}, KeyboardInterruptType: {global: true}, KeyErrorType: {global: true}, listIteratorType: {init: initListIteratorType}, ListType: {init: initListType, global: true}, LongType: {init: initLongType, global: true}, LookupErrorType: {global: true}, MemoryErrorType: {global: true}, MethodType: {init: initMethodType}, ModuleType: {init: initModuleType}, NameErrorType: {global: true}, nativeBoolMetaclassType: {init: initNativeBoolMetaclassType}, nativeFuncType: {init: initNativeFuncType}, nativeMetaclassType: {init: initNativeMetaclassType}, nativeSliceType: {init: initNativeSliceType}, nativeType: {init: initNativeType}, NoneType: {init: initNoneType, global: true}, NotImplementedErrorType: {global: true}, NotImplementedType: {init: initNotImplementedType, global: true}, ObjectType: {init: initObjectType, global: true}, OSErrorType: {global: true}, OverflowErrorType: {global: true}, PendingDeprecationWarningType: {global: true}, PropertyType: {init: initPropertyType, global: true}, rangeIteratorType: {init: initRangeIteratorType, global: true}, ReferenceErrorType: {global: true}, RuntimeErrorType: {global: true}, RuntimeWarningType: {global: true}, seqIteratorType: {init: initSeqIteratorType}, SetType: {init: initSetType, global: true}, sliceIteratorType: {init: initSliceIteratorType}, SliceType: {init: initSliceType, global: true}, StandardErrorType: {global: true}, StaticMethodType: {init: initStaticMethodType, global: true}, StopIterationType: {global: true}, StrType: {init: initStrType, global: true}, superType: {init: initSuperType, global: true}, SyntaxErrorType: {global: true}, SyntaxWarningType: {global: true}, SystemErrorType: {global: true}, SystemExitType: {global: true, init: initSystemExitType}, TracebackType: {init: initTracebackType}, TupleType: {init: initTupleType, global: true}, TypeErrorType: {global: true}, TypeType: {init: initTypeType, global: true}, UnboundLocalErrorType: {global: true}, unboundLocalType: {init: initUnboundLocalType}, UnicodeDecodeErrorType: {global: true}, UnicodeEncodeErrorType: {global: true}, UnicodeErrorType: {global: true}, UnicodeType: {init: initUnicodeType, global: true}, UnicodeWarningType: {global: true}, UserWarningType: {global: true}, ValueErrorType: {global: true}, WarningType: {global: true}, WeakRefType: {init: initWeakRefType}, xrangeType: {init: initXRangeType, global: true}, ZeroDivisionErrorType: {global: true}, } func initBuiltinType(typ *Type, info *builtinTypeInfo) { if info.state == typeStateReady { return } if info.state == typeStateInitializing { logFatal(fmt.Sprintf("cycle in type initialization for: %s", typ.name)) } info.state = typeStateInitializing for _, base := range typ.bases { baseInfo, ok := builtinTypes[base] if !ok { logFatal(fmt.Sprintf("base type not registered for: %s", typ.name)) } initBuiltinType(base, baseInfo) } prepareBuiltinType(typ, info.init) info.state = typeStateReady if typ.isSubclass(BaseExceptionType) { ExceptionTypes = append(ExceptionTypes, typ) } } func builtinAbs(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "abs", args, ObjectType); raised != nil { return nil, raised } return Abs(f, args[0]) } func builtinMapFn(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) if argc < 2 { return nil, f.RaiseType(TypeErrorType, "map() requires at least two args") } result := make([]*Object, 0, 2) z, raised := zipLongest(f, args[1:]) if raised != nil { return nil, raised } for _, tuple := range z { if args[0] == None { if argc == 2 { result = append(result, tuple[0]) } else { result = append(result, NewTuple(tuple...).ToObject()) } } else { ret, raised := args[0].Call(f, tuple, nil) if raised != nil { return nil, raised } result = append(result, ret) } } return NewList(result...).ToObject(), nil } func builtinAll(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "all", args, ObjectType); raised != nil { return nil, raised } pred := func(o *Object) (bool, *BaseException) { ret, raised := IsTrue(f, o) if raised != nil { return false, raised } return !ret, nil } foundFalseItem, raised := seqFindFirst(f, args[0], pred) if raised != nil { return nil, raised } return GetBool(!foundFalseItem).ToObject(), raised } func builtinAny(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "any", args, ObjectType); raised != nil { return nil, raised } pred := func(o *Object) (bool, *BaseException) { ret, raised := IsTrue(f, o) if raised != nil { return false, raised } return ret, nil } foundTrueItem, raised := seqFindFirst(f, args[0], pred) if raised != nil { return nil, raised } return GetBool(foundTrueItem).ToObject(), raised } func builtinBin(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "bin", args, ObjectType); raised != nil { return nil, raised } index, raised := Index(f, args[0]) if raised != nil { return nil, raised } if index == nil { format := "%s object cannot be interpreted as an index" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, args[0].typ.Name())) } return NewStr(numberToBase("0b", 2, index)).ToObject(), nil } func builtinCallable(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "callable", args, ObjectType); raised != nil { return nil, raised } o := args[0] if call := o.Type().slots.Call; call == nil { return False.ToObject(), nil } return True.ToObject(), nil } func builtinChr(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "chr", args, IntType); raised != nil { return nil, raised } i := toIntUnsafe(args[0]).Value() if i < 0 || i > 255 { return nil, f.RaiseType(ValueErrorType, "chr() arg not in range(256)") } return NewStr(string([]byte{byte(i)})).ToObject(), nil } func builtinCmp(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "cmp", args, ObjectType, ObjectType); raised != nil { return nil, raised } return Compare(f, args[0], args[1]) } func builtinDelAttr(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "delattr", args, ObjectType, StrType); raised != nil { return nil, raised } return None, DelAttr(f, args[0], toStrUnsafe(args[1])) } func builtinDir(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { // TODO: Support __dir__. if raised := checkFunctionArgs(f, "dir", args, ObjectType); raised != nil { return nil, raised } d := NewDict() o := args[0] switch { case o.isInstance(TypeType): for _, t := range toTypeUnsafe(o).mro { if raised := d.Update(f, t.Dict().ToObject()); raised != nil { return nil, raised } } case o.isInstance(ModuleType): d.Update(f, o.Dict().ToObject()) default: d = NewDict() if dict := o.Dict(); dict != nil { if raised := d.Update(f, dict.ToObject()); raised != nil { return nil, raised } } for _, t := range o.typ.mro { if raised := d.Update(f, t.Dict().ToObject()); raised != nil { return nil, raised } } } l := d.Keys(f) if raised := l.Sort(f); raised != nil { return nil, raised } return l.ToObject(), nil } func builtinDivMod(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "divmod", args, ObjectType, ObjectType); raised != nil { return nil, raised } return DivMod(f, args[0], args[1]) } func builtinFrame(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "__frame__", args); raised != nil { return nil, raised } f.taken = true return f.ToObject(), nil } func builtinGetAttr(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{ObjectType, StrType, ObjectType} argc := len(args) if argc == 2 { expectedTypes = expectedTypes[:2] } if raised := checkFunctionArgs(f, "getattr", args, expectedTypes...); raised != nil { return nil, raised } var def *Object if argc == 3 { def = args[2] } return GetAttr(f, args[0], toStrUnsafe(args[1]), def) } func builtinGlobals(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "globals", args); raised != nil { return nil, raised } return f.globals.ToObject(), nil } func builtinHasAttr(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "hasattr", args, ObjectType, StrType); raised != nil { return nil, raised } if _, raised := GetAttr(f, args[0], toStrUnsafe(args[1]), nil); raised != nil { if raised.isInstance(AttributeErrorType) { f.RestoreExc(nil, nil) return False.ToObject(), nil } return nil, raised } return True.ToObject(), nil } func builtinHash(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "hash", args, ObjectType); raised != nil { return nil, raised } h, raised := Hash(f, args[0]) if raised != nil { return nil, raised } return h.ToObject(), nil } func builtinHex(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { // In Python3 we would call __index__ similarly to builtinBin(). if raised := checkFunctionArgs(f, "hex", args, ObjectType); raised != nil { return nil, raised } return Hex(f, args[0]) } func builtinID(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "id", args, ObjectType); raised != nil { return nil, raised } return NewInt(int(uintptr(args[0].toPointer()))).ToObject(), nil } func builtinIsInstance(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "isinstance", args, ObjectType, ObjectType); raised != nil { return nil, raised } ret, raised := IsInstance(f, args[0], args[1]) if raised != nil { return nil, raised } return GetBool(ret).ToObject(), nil } func builtinIsSubclass(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "issubclass", args, ObjectType, ObjectType); raised != nil { return nil, raised } ret, raised := IsSubclass(f, args[0], args[1]) if raised != nil { return nil, raised } return GetBool(ret).ToObject(), nil } func builtinIter(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "iter", args, ObjectType); raised != nil { return nil, raised } return Iter(f, args[0]) } func builtinLen(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "len", args, ObjectType); raised != nil { return nil, raised } ret, raised := Len(f, args[0]) if raised != nil { return nil, raised } return ret.ToObject(), nil } func builtinMax(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return builtinMinMax(f, true, args, kwargs) } func builtinMin(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return builtinMinMax(f, false, args, kwargs) } func builtinNext(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "next", args, ObjectType); raised != nil { return nil, raised } ret, raised := Next(f, args[0]) if raised != nil { return nil, raised } if ret != nil { return ret, nil } return nil, f.Raise(StopIterationType.ToObject(), nil, nil) } func builtinOct(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { // In Python3 we would call __index__ similarly to builtinBin(). if raised := checkFunctionArgs(f, "oct", args, ObjectType); raised != nil { return nil, raised } return Oct(f, args[0]) } func builtinOpen(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return FileType.Call(f, args, kwargs) } func builtinOrd(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { const lenMsg = "ord() expected a character, but string of length %d found" if raised := checkFunctionArgs(f, "ord", args, BaseStringType); raised != nil { return nil, raised } o := args[0] var result int if o.isInstance(StrType) { s := toStrUnsafe(o).Value() if numChars := len(s); numChars != 1 { return nil, f.RaiseType(ValueErrorType, fmt.Sprintf(lenMsg, numChars)) } result = int(([]byte(s))[0]) } else { s := toUnicodeUnsafe(o).Value() if numChars := len(s); numChars != 1 { return nil, f.RaiseType(ValueErrorType, fmt.Sprintf(lenMsg, numChars)) } result = int(s[0]) } return NewInt(result).ToObject(), nil } func builtinPrint(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { sep := " " end := "\n" file := Stdout for _, kwarg := range kwargs { switch kwarg.Name { case "sep": kwsep, raised := ToStr(f, kwarg.Value) if raised != nil { return nil, raised } sep = kwsep.Value() case "end": kwend, raised := ToStr(f, kwarg.Value) if raised != nil { return nil, raised } end = kwend.Value() case "file": // TODO: need to map Python sys.stdout, sys.stderr etc. to os.Stdout, // os.Stderr, but for other file-like objects would need to recover // to the file descriptor probably } } return nil, pyPrint(f, args, sep, end, file) } func builtinRange(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { r, raised := xrangeType.Call(f, args, nil) if raised != nil { return nil, raised } return ListType.Call(f, []*Object{r}, nil) } func builtinRawInput(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if len(args) > 1 { msg := fmt.Sprintf("[raw_]input expcted at most 1 arguments, got %d", len(args)) return nil, f.RaiseType(TypeErrorType, msg) } if Stdin == nil { msg := fmt.Sprintf("[raw_]input: lost sys.stdin") return nil, f.RaiseType(RuntimeErrorType, msg) } if Stdout == nil { msg := fmt.Sprintf("[raw_]input: lost sys.stdout") return nil, f.RaiseType(RuntimeErrorType, msg) } if len(args) == 1 { err := pyPrint(f, args, "", "", Stdout) if err != nil { return nil, err } } line, err := Stdin.reader.ReadString('\n') if err != nil { return nil, f.RaiseType(EOFErrorType, "EOF when reading a line") } line = strings.TrimRight(line, "\n") return NewStr(line).ToObject(), nil } func builtinRepr(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "repr", args, ObjectType); raised != nil { return nil, raised } s, raised := Repr(f, args[0]) if raised != nil { return nil, raised } return s.ToObject(), nil } func builtinRound(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) expectedTypes := []*Type{ObjectType, ObjectType} if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkFunctionArgs(f, "round", args, expectedTypes...); raised != nil { return nil, raised } ndigits := 0 if argc > 1 { var raised *BaseException if ndigits, raised = IndexInt(f, args[1]); raised != nil { return nil, raised } } number, isFloat := floatCoerce(args[0]) if !isFloat { return nil, f.RaiseType(TypeErrorType, "a float is required") } if math.IsNaN(number) || math.IsInf(number, 0) || number == 0.0 { return NewFloat(number).ToObject(), nil } neg := false if number < 0 { neg = true number = -number } pow := math.Pow(10.0, float64(ndigits)) result := math.Floor(number*pow+0.5) / pow if neg { result = -result } return NewFloat(result).ToObject(), nil } func builtinSetAttr(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "setattr", args, ObjectType, StrType, ObjectType); raised != nil { return nil, raised } return None, SetAttr(f, args[0], toStrUnsafe(args[1]), args[2]) } func builtinSorted(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { // TODO: Support (cmp=None, key=None, reverse=False) if raised := checkFunctionArgs(f, "sorted", args, ObjectType); raised != nil { return nil, raised } result, raised := ListType.Call(f, Args{args[0]}, nil) if raised != nil { return nil, raised } toListUnsafe(result).Sort(f) return result, nil } func builtinSum(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) expectedTypes := []*Type{ObjectType, ObjectType} if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkFunctionArgs(f, "sum", args, expectedTypes...); raised != nil { return nil, raised } var result *Object if argc > 1 { if args[1].typ == StrType { return nil, f.RaiseType(TypeErrorType, "sum() can't sum strings [use ''.join(seq) instead]") } result = args[1] } else { result = NewInt(0).ToObject() } raised := seqForEach(f, args[0], func(o *Object) (raised *BaseException) { result, raised = Add(f, result, o) return raised }) if raised != nil { return nil, raised } return result, nil } func builtinUniChr(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "unichr", args, IntType); raised != nil { return nil, raised } i := toIntUnsafe(args[0]).Value() if i < 0 || i > unicode.MaxRune { return nil, f.RaiseType(ValueErrorType, fmt.Sprintf("unichr() arg not in range(0x%x)", unicode.MaxRune)) } return NewUnicodeFromRunes([]rune{rune(i)}).ToObject(), nil } func builtinZip(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) if argc == 0 { return NewList().ToObject(), nil } result := make([]*Object, 0, 2) iters, raised := initIters(f, args) if raised != nil { return nil, raised } Outer: for { elems := make([]*Object, argc) for i, iter := range iters { elem, raised := Next(f, iter) if raised != nil { if raised.isInstance(StopIterationType) { f.RestoreExc(nil, nil) break Outer } return nil, raised } elems[i] = elem } result = append(result, NewTuple(elems...).ToObject()) } return NewList(result...).ToObject(), nil } func init() { builtinMap := map[string]*Object{ "__debug__": False.ToObject(), "__frame__": newBuiltinFunction("__frame__", builtinFrame).ToObject(), "abs": newBuiltinFunction("abs", builtinAbs).ToObject(), "all": newBuiltinFunction("all", builtinAll).ToObject(), "any": newBuiltinFunction("any", builtinAny).ToObject(), "bin": newBuiltinFunction("bin", builtinBin).ToObject(), "callable": newBuiltinFunction("callable", builtinCallable).ToObject(), "chr": newBuiltinFunction("chr", builtinChr).ToObject(), "cmp": newBuiltinFunction("cmp", builtinCmp).ToObject(), "delattr": newBuiltinFunction("delattr", builtinDelAttr).ToObject(), "dir": newBuiltinFunction("dir", builtinDir).ToObject(), "divmod": newBuiltinFunction("divmod", builtinDivMod).ToObject(), "Ellipsis": Ellipsis, "False": False.ToObject(), "getattr": newBuiltinFunction("getattr", builtinGetAttr).ToObject(), "globals": newBuiltinFunction("globals", builtinGlobals).ToObject(), "hasattr": newBuiltinFunction("hasattr", builtinHasAttr).ToObject(), "hash": newBuiltinFunction("hash", builtinHash).ToObject(), "hex": newBuiltinFunction("hex", builtinHex).ToObject(), "id": newBuiltinFunction("id", builtinID).ToObject(), "isinstance": newBuiltinFunction("isinstance", builtinIsInstance).ToObject(), "issubclass": newBuiltinFunction("issubclass", builtinIsSubclass).ToObject(), "iter": newBuiltinFunction("iter", builtinIter).ToObject(), "len": newBuiltinFunction("len", builtinLen).ToObject(), "map": newBuiltinFunction("map", builtinMapFn).ToObject(), "max": newBuiltinFunction("max", builtinMax).ToObject(), "min": newBuiltinFunction("min", builtinMin).ToObject(), "next": newBuiltinFunction("next", builtinNext).ToObject(), "None": None, "NotImplemented": NotImplemented, "oct": newBuiltinFunction("oct", builtinOct).ToObject(), "open": newBuiltinFunction("open", builtinOpen).ToObject(), "ord": newBuiltinFunction("ord", builtinOrd).ToObject(), "print": newBuiltinFunction("print", builtinPrint).ToObject(), "range": newBuiltinFunction("range", builtinRange).ToObject(), "raw_input": newBuiltinFunction("raw_input", builtinRawInput).ToObject(), "repr": newBuiltinFunction("repr", builtinRepr).ToObject(), "round": newBuiltinFunction("round", builtinRound).ToObject(), "setattr": newBuiltinFunction("setattr", builtinSetAttr).ToObject(), "sorted": newBuiltinFunction("sorted", builtinSorted).ToObject(), "sum": newBuiltinFunction("sum", builtinSum).ToObject(), "True": True.ToObject(), "unichr": newBuiltinFunction("unichr", builtinUniChr).ToObject(), "zip": newBuiltinFunction("zip", builtinZip).ToObject(), } // Do type initialization in two phases so that we don't have to think // about hard-to-understand cycles. for typ, info := range builtinTypes { initBuiltinType(typ, info) if info.global { builtinMap[typ.name] = typ.ToObject() } } for name := range builtinMap { InternStr(name) } Builtins = newStringDict(builtinMap) } // builtinMinMax implements the builtin min/max() functions. When doMax is // true, the max is found, otherwise the min is found. There are two forms of // the builtins. The first takes a single iterable argument and the result is // the min/max of the elements of that sequence. The second form takes two or // more args and returns the min/max of those. For more details see: // https://docs.python.org/2/library/functions.html#min func builtinMinMax(f *Frame, doMax bool, args Args, kwargs KWArgs) (*Object, *BaseException) { name := "min" if doMax { name = "max" } if raised := checkFunctionVarArgs(f, name, args, ObjectType); raised != nil { return nil, raised } keyFunc := kwargs.get("key", nil) // selected is the min/max element found so far. var selected, selectedKey *Object partialFunc := func(o *Object) (raised *BaseException) { oKey := o if keyFunc != nil { oKey, raised = keyFunc.Call(f, Args{o}, nil) if raised != nil { return raised } } // sel dictates whether o is the new min/max. It defaults to // true when selected == nil (we don't yet have a selection). sel := true if selected != nil { result, raised := LT(f, selectedKey, oKey) if raised != nil { return raised } lt, raised := IsTrue(f, result) if raised != nil { return raised } // Select o when looking for max and selection < o, or // when looking for min and o < selection. sel = doMax && lt || !doMax && !lt } if sel { selected = o selectedKey = oKey } return nil } if len(args) == 1 { // Take min/max of the single iterable arg passed. if raised := seqForEach(f, args[0], partialFunc); raised != nil { return nil, raised } if selected == nil { return nil, f.RaiseType(ValueErrorType, fmt.Sprintf("%s() arg is an empty sequence", name)) } } else { // Take min/max of the passed args. for _, arg := range args { if raised := partialFunc(arg); raised != nil { return nil, raised } } } return selected, nil } // numberToBase implements the builtins "bin", "hex", and "oct". // base must be between 2 and 36, and o must be an instance of // IntType or LongType. func numberToBase(prefix string, base int, o *Object) string { z := big.Int{} switch { case o.isInstance(LongType): z = toLongUnsafe(o).value case o.isInstance(IntType): z.SetInt64(int64(toIntUnsafe(o).Value())) default: panic("numberToBase requires an Int or Long argument") } s := z.Text(base) if s[0] == '-' { // Move the negative sign before the prefix. return "-" + prefix + s[1:] } return prefix + s } // initIters return list of initiated Iter instances from the list of // iterables. func initIters(f *Frame, items []*Object) ([]*Object, *BaseException) { l := len(items) iters := make([]*Object, l) for i, arg := range items { iter, raised := Iter(f, arg) if raised != nil { return nil, raised } iters[i] = iter } return iters, nil } // zipLongest return the list of aggregates elements from each of the // iterables. If the iterables are of uneven length, missing values are // filled-in with None. func zipLongest(f *Frame, args Args) ([][]*Object, *BaseException) { argc := len(args) result := make([][]*Object, 0, 2) iters, raised := initIters(f, args) if raised != nil { return nil, raised } for { noItems := true elems := make([]*Object, argc) for i, iter := range iters { if iter == nil { continue } elem, raised := Next(f, iter) if raised != nil { if raised.isInstance(StopIterationType) { iters[i] = nil elems[i] = None f.RestoreExc(nil, nil) continue } return nil, raised } noItems = false elems[i] = elem } if noItems { break } result = append(result, elems) } return result, nil } ================================================ FILE: runtime/builtin_types_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "bytes" "fmt" "io" "math/big" "os" "testing" ) func TestBuiltinDelAttr(t *testing.T) { f := NewRootFrame() delattr := mustNotRaise(Builtins.GetItemString(f, "delattr")) fooType := newTestClass("Foo", []*Type{ObjectType}, NewDict()) fooForDelAttr := newObject(fooType) fooValue := newObject(ObjectType) mustNotRaise(nil, SetAttr(f, fooForDelAttr, NewStr("bar"), fooValue)) fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Object, *BaseException) { result, raised := delattr.Call(f, args, nil) if raised != nil { return nil, raised } val, raised := GetAttr(f, args[0], toStrUnsafe(args[1]), None) return newTestTuple(result, val == fooValue).ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(fooForDelAttr, "bar"), want: newTestTuple(None, False.ToObject()).ToObject()}, {args: wrapArgs(fooForDelAttr, "baz"), wantExc: mustCreateException(AttributeErrorType, "'Foo' object has no attribute 'baz'")}, {args: wrapArgs(fooForDelAttr), wantExc: mustCreateException(TypeErrorType, "'delattr' requires 2 arguments")}, {args: wrapArgs(fooForDelAttr, "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "'delattr' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestBuiltinFuncs(t *testing.T) { f := NewRootFrame() objectDir := ObjectType.Dict().Keys(f) objectDir.Sort(f) fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{"bar": None})) fooTypeDir := NewList(objectDir.elems...) fooTypeDir.Append(NewStr("bar").ToObject()) fooTypeDir.Sort(f) foo := newObject(fooType) SetAttr(f, foo, NewStr("baz"), None) fooDir := NewList(fooTypeDir.elems...) fooDir.Append(NewStr("baz").ToObject()) fooDir.Sort(f) dirModule := newTestModule("foo", "foo.py") if raised := dirModule.Dict().SetItemString(NewRootFrame(), "bar", newObject(ObjectType)); raised != nil { panic(raised) } dirModuleDir := dirModule.Dict().Keys(NewRootFrame()) if raised := dirModuleDir.Sort(NewRootFrame()); raised != nil { panic(raised) } iter := mustNotRaise(Iter(f, mustNotRaise(xrangeType.Call(f, wrapArgs(5), nil)))) neg := wrapFuncForTest(func(f *Frame, i int) int { return -i }) raiseKey := wrapFuncForTest(func(f *Frame, o *Object) *BaseException { return f.RaiseType(RuntimeErrorType, "foo") }) hexOctType := newTestClass("HexOct", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__hex__": newBuiltinFunction("__hex__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewStr("0xhexadecimal").ToObject(), nil }).ToObject(), "__oct__": newBuiltinFunction("__hex__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewStr("0octal").ToObject(), nil }).ToObject(), })) badNonZeroType := newTestClass("BadNonZeroType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__nonzero__": newBuiltinFunction("__nonzero__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(RuntimeErrorType, "foo") }).ToObject(), })) badNextType := newTestClass("BadNextType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "next": newBuiltinFunction("next", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(RuntimeErrorType, "foo") }).ToObject(), })) badIterType := newTestClass("BadIterType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__iter__": newBuiltinFunction("__iter__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return newObject(badNextType), nil }).ToObject(), })) addType := newTestClass("Add", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__add__": newBuiltinFunction("__add__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(1).ToObject(), nil }).ToObject(), })) fooBuiltinFunc := newBuiltinFunction("foo", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return newTestTuple(NewTuple(args.makeCopy()...), kwargs.makeDict()).ToObject(), nil }).ToObject() fooFunc := NewFunction(NewCode("foo", "foo.py", nil, CodeFlagVarArg, func(f *Frame, args []*Object) (*Object, *BaseException) { return args[0], nil }), nil) cases := []struct { f string args Args kwargs KWArgs want *Object wantExc *BaseException }{ {f: "abs", args: wrapArgs(1, 2, 3), wantExc: mustCreateException(TypeErrorType, "'abs' requires 1 arguments")}, {f: "abs", args: wrapArgs(1), want: NewInt(1).ToObject()}, {f: "abs", args: wrapArgs(-1), want: NewInt(1).ToObject()}, {f: "abs", args: wrapArgs(big.NewInt(2)), want: NewLong(big.NewInt(2)).ToObject()}, {f: "abs", args: wrapArgs(big.NewInt(-2)), want: NewLong(big.NewInt(2)).ToObject()}, {f: "abs", args: wrapArgs(NewFloat(3.4)), want: NewFloat(3.4).ToObject()}, {f: "abs", args: wrapArgs(NewFloat(-3.4)), want: NewFloat(3.4).ToObject()}, {f: "abs", args: wrapArgs(MinInt), want: NewLong(big.NewInt(MinInt).Neg(minIntBig)).ToObject()}, {f: "abs", args: wrapArgs(NewStr("a")), wantExc: mustCreateException(TypeErrorType, "bad operand type for abs(): 'str'")}, {f: "all", args: wrapArgs(newTestList()), want: True.ToObject()}, {f: "all", args: wrapArgs(newTestList(1, 2, 3)), want: True.ToObject()}, {f: "all", args: wrapArgs(newTestList(1, 0, 1)), want: False.ToObject()}, {f: "all", args: wrapArgs(13), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {f: "all", args: wrapArgs(newTestList(newObject(badNonZeroType))), wantExc: mustCreateException(RuntimeErrorType, "foo")}, {f: "all", args: wrapArgs(newObject(badIterType)), wantExc: mustCreateException(RuntimeErrorType, "foo")}, {f: "any", args: wrapArgs(newTestList()), want: False.ToObject()}, {f: "any", args: wrapArgs(newTestList(1, 2, 3)), want: True.ToObject()}, {f: "any", args: wrapArgs(newTestList(1, 0, 1)), want: True.ToObject()}, {f: "any", args: wrapArgs(newTestList(0, 0, 0)), want: False.ToObject()}, {f: "any", args: wrapArgs(newTestList(False.ToObject(), False.ToObject())), want: False.ToObject()}, {f: "any", args: wrapArgs(13), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {f: "any", args: wrapArgs(newTestList(newObject(badNonZeroType))), wantExc: mustCreateException(RuntimeErrorType, "foo")}, {f: "any", args: wrapArgs(newObject(badIterType)), wantExc: mustCreateException(RuntimeErrorType, "foo")}, {f: "bin", args: wrapArgs(64 + 8 + 1), want: NewStr("0b1001001").ToObject()}, {f: "bin", args: wrapArgs(MinInt), want: NewStr(fmt.Sprintf("-0b%b0", -(MinInt >> 1))).ToObject()}, {f: "bin", args: wrapArgs(0), want: NewStr("0b0").ToObject()}, {f: "bin", args: wrapArgs(1), want: NewStr("0b1").ToObject()}, {f: "bin", args: wrapArgs(-1), want: NewStr("-0b1").ToObject()}, {f: "bin", args: wrapArgs(big.NewInt(-1)), want: NewStr("-0b1").ToObject()}, {f: "bin", args: wrapArgs("foo"), wantExc: mustCreateException(TypeErrorType, "str object cannot be interpreted as an index")}, {f: "bin", args: wrapArgs(0.1), wantExc: mustCreateException(TypeErrorType, "float object cannot be interpreted as an index")}, {f: "bin", args: wrapArgs(1, 2, 3), wantExc: mustCreateException(TypeErrorType, "'bin' requires 1 arguments")}, {f: "bin", args: wrapArgs(newTestIndexObject(123)), want: NewStr("0b1111011").ToObject()}, {f: "callable", args: wrapArgs(fooBuiltinFunc), want: True.ToObject()}, {f: "callable", args: wrapArgs(fooFunc), want: True.ToObject()}, {f: "callable", args: wrapArgs(0), want: False.ToObject()}, {f: "callable", args: wrapArgs(0.1), want: False.ToObject()}, {f: "callable", args: wrapArgs("foo"), want: False.ToObject()}, {f: "callable", args: wrapArgs(newTestDict("foo", 1, "bar", 2)), want: False.ToObject()}, {f: "callable", args: wrapArgs(newTestList(1, 2, 3)), want: False.ToObject()}, {f: "callable", args: wrapArgs(iter), want: False.ToObject()}, {f: "callable", args: wrapArgs(1, 2), wantExc: mustCreateException(TypeErrorType, "'callable' requires 1 arguments")}, {f: "chr", args: wrapArgs(0), want: NewStr("\x00").ToObject()}, {f: "chr", args: wrapArgs(65), want: NewStr("A").ToObject()}, {f: "chr", args: wrapArgs(300), wantExc: mustCreateException(ValueErrorType, "chr() arg not in range(256)")}, {f: "chr", args: wrapArgs(-1), wantExc: mustCreateException(ValueErrorType, "chr() arg not in range(256)")}, {f: "chr", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'chr' requires 1 arguments")}, {f: "dir", args: wrapArgs(newObject(ObjectType)), want: objectDir.ToObject()}, {f: "dir", args: wrapArgs(newObject(fooType)), want: fooTypeDir.ToObject()}, {f: "dir", args: wrapArgs(fooType), want: fooTypeDir.ToObject()}, {f: "dir", args: wrapArgs(foo), want: fooDir.ToObject()}, {f: "dir", args: wrapArgs(dirModule), want: dirModuleDir.ToObject()}, {f: "dir", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'dir' requires 1 arguments")}, {f: "divmod", args: wrapArgs(12, 7), want: NewTuple2(NewInt(1).ToObject(), NewInt(5).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(-12, 7), want: NewTuple2(NewInt(-2).ToObject(), NewInt(2).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(12, -7), want: NewTuple2(NewInt(-2).ToObject(), NewInt(-2).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(-12, -7), want: NewTuple2(NewInt(1).ToObject(), NewInt(-5).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(MaxInt, MinInt), want: NewTuple2(NewInt(-1).ToObject(), NewInt(-1).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(MinInt, MaxInt), want: NewTuple2(NewInt(-2).ToObject(), NewInt(MaxInt-1).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(MinInt, -1), want: NewTuple2(NewLong(new(big.Int).Neg(minIntBig)).ToObject(), NewLong(big.NewInt(0)).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(big.NewInt(12), big.NewInt(7)), want: NewTuple2(NewLong(big.NewInt(1)).ToObject(), NewLong(big.NewInt(5)).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(big.NewInt(-12), big.NewInt(7)), want: NewTuple2(NewLong(big.NewInt(-2)).ToObject(), NewLong(big.NewInt(2)).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(big.NewInt(12), big.NewInt(-7)), want: NewTuple2(NewLong(big.NewInt(-2)).ToObject(), NewLong(big.NewInt(-2)).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(big.NewInt(-12), big.NewInt(-7)), want: NewTuple2(NewLong(big.NewInt(1)).ToObject(), NewLong(big.NewInt(-5)).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(3.25, 1.0), want: NewTuple2(NewFloat(3.0).ToObject(), NewFloat(0.25).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(-3.25, 1.0), want: NewTuple2(NewFloat(-4.0).ToObject(), NewFloat(0.75).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(3.25, -1.0), want: NewTuple2(NewFloat(-4.0).ToObject(), NewFloat(-0.75).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(-3.25, -1.0), want: NewTuple2(NewFloat(3.0).ToObject(), NewFloat(-0.25).ToObject()).ToObject()}, {f: "divmod", args: wrapArgs(NewStr("a"), NewStr("b")), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for divmod(): 'str' and 'str'")}, {f: "divmod", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'divmod' requires 2 arguments")}, {f: "getattr", args: wrapArgs(None, NewStr("foo").ToObject(), NewStr("bar").ToObject()), want: NewStr("bar").ToObject()}, {f: "getattr", args: wrapArgs(None, NewStr("foo").ToObject()), wantExc: mustCreateException(AttributeErrorType, "'NoneType' object has no attribute 'foo'")}, {f: "hasattr", args: wrapArgs(newObject(ObjectType), NewStr("foo").ToObject()), want: False.ToObject()}, {f: "hasattr", args: wrapArgs(foo, NewStr("bar").ToObject()), want: True.ToObject()}, {f: "hasattr", args: wrapArgs(foo, NewStr("baz").ToObject()), want: True.ToObject()}, {f: "hasattr", args: wrapArgs(foo, NewStr("qux").ToObject()), want: False.ToObject()}, {f: "hash", args: wrapArgs(123), want: NewInt(123).ToObject()}, {f: "hash", args: wrapArgs("foo"), want: hashFoo}, {f: "hash", args: wrapArgs(NewList()), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, {f: "hex", args: wrapArgs(0x63adbeef), want: NewStr("0x63adbeef").ToObject()}, {f: "hex", args: wrapArgs(0), want: NewStr("0x0").ToObject()}, {f: "hex", args: wrapArgs(1), want: NewStr("0x1").ToObject()}, {f: "hex", args: wrapArgs(-1), want: NewStr("-0x1").ToObject()}, {f: "hex", args: wrapArgs(big.NewInt(-1)), want: NewStr("-0x1L").ToObject()}, {f: "hex", args: wrapArgs("foo"), wantExc: mustCreateException(TypeErrorType, "hex() argument can't be converted to hex")}, {f: "hex", args: wrapArgs(0.1), wantExc: mustCreateException(TypeErrorType, "hex() argument can't be converted to hex")}, {f: "hex", args: wrapArgs(1, 2, 3), wantExc: mustCreateException(TypeErrorType, "'hex' requires 1 arguments")}, {f: "hex", args: wrapArgs(newObject(hexOctType)), want: NewStr("0xhexadecimal").ToObject()}, {f: "id", args: wrapArgs(foo), want: NewInt(int(uintptr(foo.toPointer()))).ToObject()}, {f: "id", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'id' requires 1 arguments")}, {f: "isinstance", args: wrapArgs(NewInt(42).ToObject(), IntType.ToObject()), want: True.ToObject()}, {f: "isinstance", args: wrapArgs(NewStr("foo").ToObject(), TupleType.ToObject()), want: False.ToObject()}, {f: "isinstance", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'isinstance' requires 2 arguments")}, {f: "issubclass", args: wrapArgs(IntType, IntType), want: True.ToObject()}, {f: "issubclass", args: wrapArgs(fooType, IntType), want: False.ToObject()}, {f: "issubclass", args: wrapArgs(fooType, ObjectType), want: True.ToObject()}, {f: "issubclass", args: wrapArgs(FloatType, newTestTuple(IntType, StrType)), want: False.ToObject()}, {f: "issubclass", args: wrapArgs(FloatType, newTestTuple(IntType, FloatType)), want: True.ToObject()}, {f: "issubclass", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'issubclass' requires 2 arguments")}, {f: "iter", args: wrapArgs(iter), want: iter}, {f: "iter", args: wrapArgs(42), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {f: "len", args: wrapArgs(newTestList(1, 2, 3)), want: NewInt(3).ToObject()}, {f: "len", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'len' requires 1 arguments")}, {f: "map", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "map() requires at least two args")}, {f: "map", args: wrapArgs(StrType), wantExc: mustCreateException(TypeErrorType, "map() requires at least two args")}, {f: "map", args: wrapArgs(None, newTestList()), want: newTestList().ToObject()}, {f: "map", args: wrapArgs(None, newTestList(1, 2, 3)), want: newTestList(1, 2, 3).ToObject()}, {f: "map", args: wrapArgs(None, newTestDict("foo", 1, "bar", 3)), want: newTestList("foo", "bar").ToObject()}, {f: "map", args: wrapArgs(None, None), wantExc: mustCreateException(TypeErrorType, "'NoneType' object is not iterable")}, {f: "map", args: wrapArgs(StrType, None), wantExc: mustCreateException(TypeErrorType, "'NoneType' object is not iterable")}, {f: "map", args: wrapArgs(StrType, newTestList(), None), wantExc: mustCreateException(TypeErrorType, "'NoneType' object is not iterable")}, {f: "map", args: wrapArgs(newTestList(), newTestList(1, 2, 3)), wantExc: mustCreateException(TypeErrorType, "'list' object is not callable")}, {f: "map", args: wrapArgs(StrType, newTestList()), want: newTestList().ToObject()}, {f: "map", args: wrapArgs(StrType, newTestList(1, 2, 3)), want: newTestList("1", "2", "3").ToObject()}, {f: "map", args: wrapArgs(StrType, newTestList(-1, -2, -3)), want: newTestList("-1", "-2", "-3").ToObject()}, {f: "map", args: wrapArgs(IntType, newTestList("1", "2", "3")), want: newTestList(1, 2, 3).ToObject()}, {f: "map", args: wrapArgs(IntType, newTestList("-1", "-2", "-3")), want: newTestList(-1, -2, -3).ToObject()}, {f: "map", args: wrapArgs(IntType, "123"), want: newTestList(1, 2, 3).ToObject()}, {f: "map", args: wrapArgs(IntType, newTestDict("1", "11", "2", "22")), want: newTestList(1, 2).ToObject()}, {f: "map", args: wrapArgs(IntType, 1), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {f: "map", args: wrapArgs(1, newTestList(1, 2, 3)), wantExc: mustCreateException(TypeErrorType, "'int' object is not callable")}, {f: "map", args: wrapArgs(StrType, newTestList(), 1), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {f: "max", args: wrapArgs(2, 3, 1), want: NewInt(3).ToObject()}, {f: "max", args: wrapArgs("bar", "foo"), want: NewStr("foo").ToObject()}, {f: "max", args: wrapArgs(newTestList(2, 3, 1)), want: NewInt(3).ToObject()}, {f: "max", args: wrapArgs(newTestList("bar", "foo")), want: NewStr("foo").ToObject()}, {f: "max", args: wrapArgs(2, 3, 1), want: NewInt(3).ToObject()}, {f: "max", args: wrapArgs("bar", "foo"), want: NewStr("foo").ToObject()}, {f: "max", args: wrapArgs(newTestList(2, 3, 1)), want: NewInt(3).ToObject()}, {f: "max", args: wrapArgs(newTestList("bar", "foo")), want: NewStr("foo").ToObject()}, {f: "max", args: wrapArgs(2, 3, 1), kwargs: wrapKWArgs("key", neg), want: NewInt(1).ToObject()}, {f: "max", args: wrapArgs(1, 2, 3), kwargs: wrapKWArgs("key", neg), want: NewInt(1).ToObject()}, {f: "max", args: wrapArgs(newTestList(2, 3, 1)), kwargs: wrapKWArgs("key", neg), want: NewInt(1).ToObject()}, {f: "max", args: wrapArgs(newTestList(1, 2, 3)), kwargs: wrapKWArgs("key", neg), want: NewInt(1).ToObject()}, {f: "max", args: wrapArgs(2, 3, 1), kwargs: wrapKWArgs("key", neg), want: NewInt(1).ToObject()}, {f: "max", args: wrapArgs(1, 2, 3), kwargs: wrapKWArgs("key", neg), want: NewInt(1).ToObject()}, {f: "max", args: wrapArgs(newTestList(2, 3, 1)), kwargs: wrapKWArgs("key", neg), want: NewInt(1).ToObject()}, {f: "max", args: wrapArgs(newTestList(1, 2, 3)), kwargs: wrapKWArgs("key", neg), want: NewInt(1).ToObject()}, {f: "max", args: wrapArgs(newTestList("foo")), want: NewStr("foo").ToObject()}, {f: "max", args: wrapArgs(1), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {f: "max", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'max' requires 1 arguments")}, {f: "max", args: wrapArgs(newTestList()), wantExc: mustCreateException(ValueErrorType, "max() arg is an empty sequence")}, {f: "max", args: wrapArgs(1, 2), kwargs: wrapKWArgs("key", raiseKey), wantExc: mustCreateException(RuntimeErrorType, "foo")}, {f: "min", args: wrapArgs(2, 3, 1), want: NewInt(1).ToObject()}, {f: "min", args: wrapArgs("bar", "foo"), want: NewStr("bar").ToObject()}, {f: "min", args: wrapArgs(newTestList(2, 3, 1)), want: NewInt(1).ToObject()}, {f: "min", args: wrapArgs(newTestList("bar", "foo")), want: NewStr("bar").ToObject()}, {f: "min", args: wrapArgs(2, 3, 1), want: NewInt(1).ToObject()}, {f: "min", args: wrapArgs("bar", "foo"), want: NewStr("bar").ToObject()}, {f: "min", args: wrapArgs(newTestList(2, 3, 1)), want: NewInt(1).ToObject()}, {f: "min", args: wrapArgs(newTestList("bar", "foo")), want: NewStr("bar").ToObject()}, {f: "min", args: wrapArgs(2, 3, 1), kwargs: wrapKWArgs("key", neg), want: NewInt(3).ToObject()}, {f: "min", args: wrapArgs(1, 2, 3), kwargs: wrapKWArgs("key", neg), want: NewInt(3).ToObject()}, {f: "min", args: wrapArgs(newTestList(2, 3, 1)), kwargs: wrapKWArgs("key", neg), want: NewInt(3).ToObject()}, {f: "min", args: wrapArgs(newTestList(1, 2, 3)), kwargs: wrapKWArgs("key", neg), want: NewInt(3).ToObject()}, {f: "min", args: wrapArgs(2, 3, 1), kwargs: wrapKWArgs("key", neg), want: NewInt(3).ToObject()}, {f: "min", args: wrapArgs(1, 2, 3), kwargs: wrapKWArgs("key", neg), want: NewInt(3).ToObject()}, {f: "min", args: wrapArgs(newTestList(2, 3, 1)), kwargs: wrapKWArgs("key", neg), want: NewInt(3).ToObject()}, {f: "min", args: wrapArgs(newTestList(1, 2, 3)), kwargs: wrapKWArgs("key", neg), want: NewInt(3).ToObject()}, {f: "min", args: wrapArgs(newTestList("foo")), want: NewStr("foo").ToObject()}, {f: "min", args: wrapArgs(1), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {f: "min", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'min' requires 1 arguments")}, {f: "min", args: wrapArgs(newTestList()), wantExc: mustCreateException(ValueErrorType, "min() arg is an empty sequence")}, {f: "min", args: wrapArgs(1, 2), kwargs: wrapKWArgs("key", raiseKey), wantExc: mustCreateException(RuntimeErrorType, "foo")}, {f: "oct", args: wrapArgs(077), want: NewStr("077").ToObject()}, {f: "oct", args: wrapArgs(0), want: NewStr("0").ToObject()}, {f: "oct", args: wrapArgs(1), want: NewStr("01").ToObject()}, {f: "oct", args: wrapArgs(-1), want: NewStr("-01").ToObject()}, {f: "oct", args: wrapArgs(big.NewInt(-1)), want: NewStr("-01L").ToObject()}, {f: "oct", args: wrapArgs("foo"), wantExc: mustCreateException(TypeErrorType, "oct() argument can't be converted to oct")}, {f: "oct", args: wrapArgs(0.1), wantExc: mustCreateException(TypeErrorType, "oct() argument can't be converted to oct")}, {f: "oct", args: wrapArgs(1, 2, 3), wantExc: mustCreateException(TypeErrorType, "'oct' requires 1 arguments")}, {f: "oct", args: wrapArgs(newObject(hexOctType)), want: NewStr("0octal").ToObject()}, {f: "ord", args: wrapArgs("a"), want: NewInt(97).ToObject()}, {f: "ord", args: wrapArgs(NewUnicode("樂")), want: NewInt(63764).ToObject()}, {f: "ord", args: wrapArgs("foo"), wantExc: mustCreateException(ValueErrorType, "ord() expected a character, but string of length 3 found")}, {f: "ord", args: wrapArgs(NewUnicode("волн")), wantExc: mustCreateException(ValueErrorType, "ord() expected a character, but string of length 4 found")}, {f: "ord", args: wrapArgs(1, 2, 3), wantExc: mustCreateException(TypeErrorType, "'ord' requires 1 arguments")}, {f: "range", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'__new__' of 'int' requires 3 arguments")}, {f: "range", args: wrapArgs(3), want: newTestList(0, 1, 2).ToObject()}, {f: "range", args: wrapArgs(10, 0), want: NewList().ToObject()}, {f: "range", args: wrapArgs(-12, -23, -5), want: newTestList(-12, -17, -22).ToObject()}, {f: "repr", args: wrapArgs(123), want: NewStr("123").ToObject()}, {f: "repr", args: wrapArgs(NewUnicode("abc")), want: NewStr("u'abc'").ToObject()}, {f: "repr", args: wrapArgs(newTestTuple("foo", "bar")), want: NewStr("('foo', 'bar')").ToObject()}, {f: "repr", args: wrapArgs("a", "b", "c"), wantExc: mustCreateException(TypeErrorType, "'repr' requires 1 arguments")}, {f: "round", args: wrapArgs(1234.567), want: NewFloat(1235).ToObject()}, {f: "round", args: wrapArgs(1234.111), want: NewFloat(1234).ToObject()}, {f: "round", args: wrapArgs(-1234.567), want: NewFloat(-1235).ToObject()}, {f: "round", args: wrapArgs(-1234.111), want: NewFloat(-1234).ToObject()}, {f: "round", args: wrapArgs(1234.567, newTestIndexObject(0)), want: NewFloat(1235).ToObject()}, {f: "round", args: wrapArgs("foo"), wantExc: mustCreateException(TypeErrorType, "a float is required")}, {f: "round", args: wrapArgs(12.5, 0), want: NewFloat(13.0).ToObject()}, {f: "round", args: wrapArgs(-12.5, 0), want: NewFloat(-13.0).ToObject()}, {f: "round", args: wrapArgs(12.5, 3), want: NewFloat(12.5).ToObject()}, {f: "round", args: wrapArgs(1234.5, 1), want: NewFloat(1234.5).ToObject()}, {f: "round", args: wrapArgs(1234.5, 1), want: NewFloat(1234.5).ToObject()}, {f: "round", args: wrapArgs(1234.56, 1), want: NewFloat(1234.6).ToObject()}, {f: "round", args: wrapArgs(-1234.56, 1), want: NewFloat(-1234.6).ToObject()}, {f: "round", args: wrapArgs(-1234.56, -2), want: NewFloat(-1200.0).ToObject()}, {f: "round", args: wrapArgs(-1234.56, -8), want: NewFloat(0.0).ToObject()}, {f: "round", args: wrapArgs(63.4, -3), want: NewFloat(0.0).ToObject()}, {f: "round", args: wrapArgs(63.4, -2), want: NewFloat(100.0).ToObject()}, {f: "sorted", args: wrapArgs(NewList()), want: NewList().ToObject()}, {f: "sorted", args: wrapArgs(newTestList("foo", "bar")), want: newTestList("bar", "foo").ToObject()}, {f: "sorted", args: wrapArgs(newTestList(true, false)), want: newTestList(false, true).ToObject()}, {f: "sorted", args: wrapArgs(newTestList(1, 2, 0, 3)), want: newTestRange(4).ToObject()}, {f: "sorted", args: wrapArgs(newTestRange(100)), want: newTestRange(100).ToObject()}, {f: "sorted", args: wrapArgs(newTestTuple(1, 2, 0, 3)), want: newTestRange(4).ToObject()}, {f: "sorted", args: wrapArgs(newTestDict("foo", 1, "bar", 2)), want: newTestList("bar", "foo").ToObject()}, {f: "sorted", args: wrapArgs(1), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {f: "sorted", args: wrapArgs(newTestList("foo", "bar"), 2), wantExc: mustCreateException(TypeErrorType, "'sorted' requires 1 arguments")}, {f: "sum", args: wrapArgs(newTestList(1, 2, 3, 4)), want: NewInt(10).ToObject()}, {f: "sum", args: wrapArgs(newTestList(1, 2), 3), want: NewFloat(6).ToObject()}, {f: "sum", args: wrapArgs(newTestList(2, 1.1)), want: NewFloat(3.1).ToObject()}, {f: "sum", args: wrapArgs(newTestList(2, 1.1, 2)), want: NewFloat(5.1).ToObject()}, {f: "sum", args: wrapArgs(newTestList(2, 1.1, 2.0)), want: NewFloat(5.1).ToObject()}, {f: "sum", args: wrapArgs(newTestList(1), newObject(addType)), want: NewInt(1).ToObject()}, {f: "sum", args: wrapArgs(newTestList(newObject(addType)), newObject(addType)), want: NewInt(1).ToObject()}, {f: "unichr", args: wrapArgs(0), want: NewUnicode("\x00").ToObject()}, {f: "unichr", args: wrapArgs(65), want: NewStr("A").ToObject()}, {f: "unichr", args: wrapArgs(0x120000), wantExc: mustCreateException(ValueErrorType, "unichr() arg not in range(0x10ffff)")}, {f: "unichr", args: wrapArgs(-1), wantExc: mustCreateException(ValueErrorType, "unichr() arg not in range(0x10ffff)")}, {f: "unichr", args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'unichr' requires 1 arguments")}, {f: "zip", args: wrapArgs(), want: newTestList().ToObject()}, {f: "zip", args: wrapArgs(newTestTuple()), want: newTestList().ToObject()}, {f: "zip", args: wrapArgs(newTestList()), want: newTestList().ToObject()}, {f: "zip", args: wrapArgs(newTestList(1)), want: newTestList(newTestTuple(1).ToObject()).ToObject()}, {f: "zip", args: wrapArgs(newTestList(1, 2, 3)), want: newTestList(newTestTuple(1).ToObject(), newTestTuple(2).ToObject(), newTestTuple(3).ToObject()).ToObject()}, {f: "zip", args: wrapArgs(newTestRange(3)), want: newTestList(newTestTuple(0).ToObject(), newTestTuple(1).ToObject(), newTestTuple(2).ToObject()).ToObject()}, {f: "zip", args: wrapArgs(newTestTuple(1, 2, 3), newTestTuple(4, 5, 6)), want: NewList(newTestTuple(1, 4).ToObject(), newTestTuple(2, 5).ToObject(), newTestTuple(3, 6).ToObject()).ToObject()}, {f: "zip", args: wrapArgs(newTestTuple(1, 2, 3), newTestTuple(4, 5)), want: NewList(newTestTuple(1, 4).ToObject(), newTestTuple(2, 5).ToObject()).ToObject()}, {f: "zip", args: wrapArgs(newTestTuple(1, 2), newTestTuple(4, 5, 5)), want: NewList(newTestTuple(1, 4).ToObject(), newTestTuple(2, 5).ToObject()).ToObject()}, {f: "zip", args: wrapArgs(1), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {f: "zip", args: wrapArgs(newTestDict("foo", 1, "bar", 2)), want: newTestList(newTestTuple("foo").ToObject(), newTestTuple("bar").ToObject()).ToObject()}, } for _, cas := range cases { fun := mustNotRaise(Builtins.GetItemString(NewRootFrame(), cas.f)) if fun == nil { t.Fatalf("%s not found in builtins: %v", cas.f, Builtins) } testCase := invokeTestCase{args: cas.args, kwargs: cas.kwargs, want: cas.want, wantExc: cas.wantExc} if err := runInvokeTestCase(fun, &testCase); err != "" { t.Error(err) } } } func TestBuiltinGlobals(t *testing.T) { f := NewRootFrame() f.globals = newTestDict("foo", 1, "bar", 2, 42, None) globals := mustNotRaise(Builtins.GetItemString(f, "globals")) got, raised := globals.Call(f, nil, nil) want := newTestDict("foo", 1, "bar", 2, 42, None).ToObject() switch checkResult(got, want, raised, nil) { case checkInvokeResultExceptionMismatch: t.Errorf("globals() = %v, want %v", got, want) case checkInvokeResultReturnValueMismatch: t.Errorf("globals() raised %v, want nil", raised) } } func TestEllipsisRepr(t *testing.T) { cas := invokeTestCase{args: wrapArgs(Ellipsis), want: NewStr("Ellipsis").ToObject()} if err := runInvokeMethodTestCase(EllipsisType, "__repr__", &cas); err != "" { t.Error(err) } } func TestNoneRepr(t *testing.T) { cas := invokeTestCase{args: wrapArgs(None), want: NewStr("None").ToObject()} if err := runInvokeMethodTestCase(NoneType, "__repr__", &cas); err != "" { t.Error(err) } } func TestNotImplementedRepr(t *testing.T) { cas := invokeTestCase{args: wrapArgs(NotImplemented), want: NewStr("NotImplemented").ToObject()} if err := runInvokeMethodTestCase(NotImplementedType, "__repr__", &cas); err != "" { t.Error(err) } } // captureStdout invokes a function closure which writes to stdout and captures // its output as string. func captureStdout(f *Frame, fn func() *BaseException) (string, *BaseException) { r, w, err := os.Pipe() if err != nil { return "", f.RaiseType(RuntimeErrorType, fmt.Sprintf("failed to open pipe: %v", err)) } oldStdout := Stdout Stdout = NewFileFromFD(w.Fd(), nil) defer func() { Stdout = oldStdout }() done := make(chan struct{}) var raised *BaseException go func() { defer close(done) defer w.Close() raised = fn() }() var buf bytes.Buffer if _, err := io.Copy(&buf, r); err != nil { return "", f.RaiseType(RuntimeErrorType, fmt.Sprintf("failed to copy buffer: %v", err)) } <-done if raised != nil { return "", raised } return buf.String(), nil } // TODO(corona10): Re-enable once #282 is addressed. /*func TestBuiltinPrint(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, args *Tuple, kwargs KWArgs) (string, *BaseException) { return captureStdout(f, func() *BaseException { _, raised := builtinPrint(NewRootFrame(), args.elems, kwargs) return raised }) }) cases := []invokeTestCase{ {args: wrapArgs(NewTuple(), wrapKWArgs()), want: NewStr("\n").ToObject()}, {args: wrapArgs(newTestTuple("abc"), wrapKWArgs()), want: NewStr("abc\n").ToObject()}, {args: wrapArgs(newTestTuple("abc", 123), wrapKWArgs()), want: NewStr("abc 123\n").ToObject()}, {args: wrapArgs(newTestTuple("abc", 123), wrapKWArgs("sep", "")), want: NewStr("abc123\n").ToObject()}, {args: wrapArgs(newTestTuple("abc", 123), wrapKWArgs("end", "")), want: NewStr("abc 123").ToObject()}, {args: wrapArgs(newTestTuple("abc", 123), wrapKWArgs("sep", "XX", "end", "--")), want: NewStr("abcXX123--").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } }*/ func TestBuiltinSetAttr(t *testing.T) { setattr := mustNotRaise(Builtins.GetItemString(NewRootFrame(), "setattr")) fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{})) foo := newObject(fooType) fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Object, *BaseException) { result, raised := setattr.Call(f, args, nil) if raised != nil { return nil, raised } val, raised := GetAttr(f, args[0], toStrUnsafe(args[1]), nil) if raised != nil { return nil, raised } return newTestTuple(result, val).ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(foo), wantExc: mustCreateException(TypeErrorType, "'setattr' requires 3 arguments")}, {args: wrapArgs(newObject(fooType), "foo", "bar"), want: newTestTuple(None, "bar").ToObject()}, {args: wrapArgs(newObject(fooType), "foo", 123), want: newTestTuple(None, 123).ToObject()}, {args: wrapArgs(foo, "foo"), wantExc: mustCreateException(TypeErrorType, "'setattr' requires 3 arguments")}, {args: wrapArgs(foo, "foo", 123, None), wantExc: mustCreateException(TypeErrorType, "'setattr' requires 3 arguments")}, {args: wrapArgs(foo, 123, 123), wantExc: mustCreateException(TypeErrorType, "'setattr' requires a 'str' object but received a \"int\"")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } // TODO(corona10): Re-enable once #282 is addressed. /*func TestRawInput(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, s string, args ...*Object) (*Object, *BaseException) { // Create a fake Stdin for input test. stdinFile, w, err := os.Pipe() if err != nil { return nil, f.RaiseType(RuntimeErrorType, fmt.Sprintf("failed to open pipe: %v", err)) } go func() { w.Write([]byte(s)) w.Close() }() oldStdin := Stdin Stdin = NewFileFromFD(stdinFile.Fd(), nil) defer func() { Stdin = oldStdin stdinFile.Close() }() var input *Object output, raised := captureStdout(f, func() *BaseException { in, raised := builtinRawInput(f, args, nil) input = in return raised }) if raised != nil { return nil, raised } return newTestTuple(input, output).ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs("HelloGrumpy\n", ""), want: newTestTuple("HelloGrumpy", "").ToObject()}, {args: wrapArgs("HelloGrumpy\n", "ShouldBeShown\nShouldBeShown\t"), want: newTestTuple("HelloGrumpy", "ShouldBeShown\nShouldBeShown\t").ToObject()}, {args: wrapArgs("HelloGrumpy\n", 5, 4), wantExc: mustCreateException(TypeErrorType, "[raw_]input expcted at most 1 arguments, got 2")}, {args: wrapArgs("HelloGrumpy\nHelloGrumpy\n", ""), want: newTestTuple("HelloGrumpy", "").ToObject()}, {args: wrapArgs("HelloGrumpy\nHelloGrumpy\n", "ShouldBeShown\nShouldBeShown\t"), want: newTestTuple("HelloGrumpy", "ShouldBeShown\nShouldBeShown\t").ToObject()}, {args: wrapArgs("HelloGrumpy\nHelloGrumpy\n", 5, 4), wantExc: mustCreateException(TypeErrorType, "[raw_]input expcted at most 1 arguments, got 2")}, {args: wrapArgs("", ""), wantExc: mustCreateException(EOFErrorType, "EOF when reading a line")}, {args: wrapArgs("", "ShouldBeShown\nShouldBeShown\t"), wantExc: mustCreateException(EOFErrorType, "EOF when reading a line")}, {args: wrapArgs("", 5, 4), wantExc: mustCreateException(TypeErrorType, "[raw_]input expcted at most 1 arguments, got 2")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } }*/ func newTestIndexObject(index int) *Object { indexType := newTestClass("Index", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(index).ToObject(), nil }).ToObject(), })) return newObject(indexType) } ================================================ FILE: runtime/bytearray.go ================================================ // Copyright 2017 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "bytes" "fmt" "reflect" "sync" ) var ( // ByteArrayType is the object representing the Python 'bytearray' type. ByteArrayType = newBasisType("bytearray", reflect.TypeOf(ByteArray{}), toByteArrayUnsafe, ObjectType) ) // ByteArray represents Python 'bytearray' objects. type ByteArray struct { Object mutex sync.RWMutex value []byte } func toByteArrayUnsafe(o *Object) *ByteArray { return (*ByteArray)(o.toPointer()) } // ToObject upcasts a to an Object. func (a *ByteArray) ToObject() *Object { return &a.Object } // Value returns the underlying bytes held by a. func (a *ByteArray) Value() []byte { return a.value } func byteArrayEq(f *Frame, v, w *Object) (*Object, *BaseException) { return byteArrayCompare(v, w, False, True, False), nil } func byteArrayGE(f *Frame, v, w *Object) (*Object, *BaseException) { return byteArrayCompare(v, w, False, True, True), nil } func byteArrayGetItem(f *Frame, o, key *Object) (result *Object, raised *BaseException) { a := toByteArrayUnsafe(o) if key.typ.slots.Index != nil { index, raised := IndexInt(f, key) if raised != nil { return nil, raised } a.mutex.RLock() elems := a.Value() if index, raised = seqCheckedIndex(f, len(elems), index); raised == nil { result = NewInt(int(elems[index])).ToObject() } a.mutex.RUnlock() return result, raised } if key.isInstance(SliceType) { a.mutex.RLock() elems := a.Value() start, stop, step, sliceLen, raised := toSliceUnsafe(key).calcSlice(f, len(elems)) if raised == nil { value := make([]byte, sliceLen) if step == 1 { copy(value, elems[start:stop]) } else { i := 0 for j := start; j != stop; j += step { value[i] = elems[j] i++ } } result = (&ByteArray{Object: Object{typ: ByteArrayType}, value: value}).ToObject() } a.mutex.RUnlock() return result, raised } return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("bytearray indices must be integers or slice, not %s", key.typ.Name())) } func byteArrayGT(f *Frame, v, w *Object) (*Object, *BaseException) { return byteArrayCompare(v, w, False, False, True), nil } func byteArrayInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "__init__", args, IntType); raised != nil { return nil, raised } a := toByteArrayUnsafe(o) a.mutex.Lock() a.value = make([]byte, toIntUnsafe(args[0]).Value()) a.mutex.Unlock() return None, nil } func byteArrayLE(f *Frame, v, w *Object) (*Object, *BaseException) { return byteArrayCompare(v, w, True, True, False), nil } func byteArrayLT(f *Frame, v, w *Object) (*Object, *BaseException) { return byteArrayCompare(v, w, True, False, False), nil } func byteArrayNative(f *Frame, o *Object) (reflect.Value, *BaseException) { a := toByteArrayUnsafe(o) a.mutex.RLock() result := reflect.ValueOf(a.Value()) a.mutex.RUnlock() return result, nil } func byteArrayNE(f *Frame, v, w *Object) (*Object, *BaseException) { return byteArrayCompare(v, w, True, False, True), nil } func byteArrayRepr(f *Frame, o *Object) (*Object, *BaseException) { a := toByteArrayUnsafe(o) a.mutex.RLock() s, raised := Repr(f, NewStr(string(a.Value())).ToObject()) a.mutex.RUnlock() if raised != nil { return nil, raised } return NewStr(fmt.Sprintf("bytearray(b%s)", s.Value())).ToObject(), nil } func byteArrayStr(f *Frame, o *Object) (*Object, *BaseException) { a := toByteArrayUnsafe(o) a.mutex.RLock() s := string(a.Value()) a.mutex.RUnlock() return NewStr(s).ToObject(), nil } func initByteArrayType(dict map[string]*Object) { ByteArrayType.slots.Eq = &binaryOpSlot{byteArrayEq} ByteArrayType.slots.GE = &binaryOpSlot{byteArrayGE} ByteArrayType.slots.GetItem = &binaryOpSlot{byteArrayGetItem} ByteArrayType.slots.GT = &binaryOpSlot{byteArrayGT} ByteArrayType.slots.Init = &initSlot{byteArrayInit} ByteArrayType.slots.LE = &binaryOpSlot{byteArrayLE} ByteArrayType.slots.LT = &binaryOpSlot{byteArrayLT} ByteArrayType.slots.Native = &nativeSlot{byteArrayNative} ByteArrayType.slots.NE = &binaryOpSlot{byteArrayNE} ByteArrayType.slots.Repr = &unaryOpSlot{byteArrayRepr} ByteArrayType.slots.Str = &unaryOpSlot{byteArrayStr} } func byteArrayCompare(v, w *Object, ltResult, eqResult, gtResult *Int) *Object { if v == w { return eqResult.ToObject() } // For simplicity we make a copy of w if it's a str or bytearray. This // is inefficient and it may be useful to optimize. var data []byte switch { case w.isInstance(StrType): data = []byte(toStrUnsafe(w).Value()) case w.isInstance(ByteArrayType): a := toByteArrayUnsafe(w) a.mutex.RLock() data = make([]byte, len(a.value)) copy(data, a.value) a.mutex.RUnlock() default: return NotImplemented } a := toByteArrayUnsafe(v) a.mutex.RLock() cmp := bytes.Compare(a.value, data) a.mutex.RUnlock() switch cmp { case -1: return ltResult.ToObject() case 0: return eqResult.ToObject() default: return gtResult.ToObject() } } ================================================ FILE: runtime/bytearray_test.go ================================================ // Copyright 2017 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestByteArrayCompare(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestByteArray(""), newTestByteArray("")), want: compareAllResultEq}, {args: wrapArgs(newTestByteArray("foo"), newTestByteArray("foo")), want: compareAllResultEq}, {args: wrapArgs(newTestByteArray(""), newTestByteArray("foo")), want: compareAllResultLT}, {args: wrapArgs(newTestByteArray("foo"), newTestByteArray("")), want: compareAllResultGT}, {args: wrapArgs(newTestByteArray("bar"), newTestByteArray("baz")), want: compareAllResultLT}, {args: wrapArgs(newTestByteArray(""), ""), want: compareAllResultEq}, {args: wrapArgs(newTestByteArray("foo"), "foo"), want: compareAllResultEq}, {args: wrapArgs(newTestByteArray(""), "foo"), want: compareAllResultLT}, {args: wrapArgs(newTestByteArray("foo"), ""), want: compareAllResultGT}, {args: wrapArgs(newTestByteArray("bar"), "baz"), want: compareAllResultLT}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestByteArrayGetItem(t *testing.T) { badIndexType := newTestClass("badIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(ValueErrorType, "wut") }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(newTestByteArray("bar"), 1), want: NewInt(97).ToObject()}, {args: wrapArgs(newTestByteArray("foo"), 3.14), wantExc: mustCreateException(TypeErrorType, "bytearray indices must be integers or slice, not float")}, {args: wrapArgs(newTestByteArray("baz"), -1), want: NewInt(122).ToObject()}, {args: wrapArgs(newTestByteArray("baz"), -4), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(newTestByteArray(""), 0), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(newTestByteArray("foo"), 3), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(newTestByteArray("bar"), newTestSlice(None, 2)), want: newTestByteArray("ba").ToObject()}, {args: wrapArgs(newTestByteArray("bar"), newTestSlice(1, 3)), want: newTestByteArray("ar").ToObject()}, {args: wrapArgs(newTestByteArray("bar"), newTestSlice(1, None)), want: newTestByteArray("ar").ToObject()}, {args: wrapArgs(newTestByteArray("foobarbaz"), newTestSlice(1, 8, 2)), want: newTestByteArray("obra").ToObject()}, {args: wrapArgs(newTestByteArray("abc"), newTestSlice(None, None, -1)), want: newTestByteArray("cba").ToObject()}, {args: wrapArgs(newTestByteArray("bar"), newTestSlice(1, 2, 0)), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, {args: wrapArgs(newTestByteArray("123"), newObject(badIndexType)), wantExc: mustCreateException(ValueErrorType, "wut")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(ByteArrayType, "__getitem__", &cas); err != "" { t.Error(err) } } } func TestByteArrayInit(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(3), want: newTestByteArray("\x00\x00\x00").ToObject()}, {args: wrapArgs(newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, `'__init__' requires a 'int' object but received a "object"`)}, } for _, cas := range cases { if err := runInvokeTestCase(ByteArrayType.ToObject(), &cas); err != "" { t.Error(err) } } } func TestByteArrayNative(t *testing.T) { val, raised := ToNative(NewRootFrame(), newTestByteArray("foo").ToObject()) if raised != nil { t.Fatalf("bytearray.__native__ raised %v", raised) } data, ok := val.Interface().([]byte) if !ok || string(data) != "foo" { t.Fatalf(`bytearray.__native__() = %v, want []byte("123")`, val.Interface()) } } func TestByteArrayRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestByteArray("")), want: NewStr("bytearray(b'')").ToObject()}, {args: wrapArgs(newTestByteArray("foo")), want: NewStr("bytearray(b'foo')").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func TestByteArrayStr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestByteArray("")), want: NewStr("").ToObject()}, {args: wrapArgs(newTestByteArray("foo")), want: NewStr("foo").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } } } func newTestByteArray(s string) *ByteArray { return &ByteArray{Object: Object{typ: ByteArrayType}, value: []byte(s)} } ================================================ FILE: runtime/code.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "reflect" ) // CodeType is the object representing the Python 'code' type. var CodeType = newBasisType("code", reflect.TypeOf(Code{}), toCodeUnsafe, ObjectType) // CodeFlag is a switch controlling the behavior of a Code object. type CodeFlag int const ( // CodeFlagVarArg means a Code object accepts *arg parameters. CodeFlagVarArg CodeFlag = 4 // CodeFlagKWArg means a Code object accepts **kwarg parameters. CodeFlagKWArg CodeFlag = 8 ) // Code represents Python 'code' objects. type Code struct { Object name string `attr:"co_name"` filename string `attr:"co_filename"` // argc is the number of positional arguments. argc int `attr:"co_argcount"` flags CodeFlag `attr:"co_flags"` paramSpec *ParamSpec fn func(*Frame, []*Object) (*Object, *BaseException) } // NewCode creates a new Code object that executes the given fn. func NewCode(name, filename string, params []Param, flags CodeFlag, fn func(*Frame, []*Object) (*Object, *BaseException)) *Code { s := NewParamSpec(name, params, flags&CodeFlagVarArg != 0, flags&CodeFlagKWArg != 0) return &Code{Object{typ: CodeType}, name, filename, len(params), flags, s, fn} } func toCodeUnsafe(o *Object) *Code { return (*Code)(o.toPointer()) } // Eval runs the code object c in the context of the given globals. func (c *Code) Eval(f *Frame, globals *Dict, args Args, kwargs KWArgs) (*Object, *BaseException) { validated := f.MakeArgs(c.paramSpec.Count) if raised := c.paramSpec.Validate(f, validated, args, kwargs); raised != nil { return nil, raised } oldExc, oldTraceback := f.ExcInfo() next := newChildFrame(f) next.code = c next.globals = globals ret, raised := c.fn(next, validated) next.release() f.FreeArgs(validated) if raised == nil { // Restore exc_info to what it was when we left the previous // frame. f.RestoreExc(oldExc, oldTraceback) if ret == nil { ret = None } } else { _, tb := f.ExcInfo() if f.code != nil { // The root frame has no code object so don't include it // in the traceback. tb = newTraceback(f, tb) } f.RestoreExc(raised, tb) } return ret, raised } ================================================ FILE: runtime/code_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestXxx(t *testing.T) { } func TestNewCodeKeywordsCheck(t *testing.T) { oldLogFatal := logFatal defer func() { logFatal = oldLogFatal }() var got string logFatal = func(msg string) { got = msg } NewCode("foo", "foo.py", []Param{{"bar", None}, {"baz", nil}}, 0, nil) if want := "foo() non-keyword arg baz after keyword arg"; got != want { t.Errorf("NewCode logged %q, want %q", got, want) } } func TestNewCode(t *testing.T) { testFunc := newBuiltinFunction("TestNewCode", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionVarArgs(f, "TestNewCode", args, CodeType); raised != nil { return nil, raised } return toCodeUnsafe(args[0]).Eval(f, nil, args[1:], kwargs) }) fn := func(f *Frame, args []*Object) (*Object, *BaseException) { return NewTuple(Args(args).makeCopy()...).ToObject(), nil } nilFn := func(*Frame, []*Object) (*Object, *BaseException) { return nil, nil } cases := []invokeTestCase{ invokeTestCase{args: wrapArgs(NewCode("f1", "foo.py", nil, 0, fn)), want: NewTuple().ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f2", "foo.py", []Param{{"a", nil}}, 0, fn), 123), want: newTestTuple(123).ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f2", "foo.py", []Param{{"a", nil}}, 0, fn)), kwargs: wrapKWArgs("a", "apple"), want: newTestTuple("apple").ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f2", "foo.py", []Param{{"a", nil}}, 0, fn)), kwargs: wrapKWArgs("b", "bear"), wantExc: mustCreateException(TypeErrorType, "f2() got an unexpected keyword argument 'b'")}, invokeTestCase{args: wrapArgs(NewCode("f2", "foo.py", []Param{{"a", nil}}, 0, fn)), wantExc: mustCreateException(TypeErrorType, "f2() takes at least 1 arguments (0 given)")}, invokeTestCase{args: wrapArgs(NewCode("f2", "foo.py", []Param{{"a", nil}}, 0, fn), 1, 2, 3), wantExc: mustCreateException(TypeErrorType, "f2() takes 1 arguments (3 given)")}, invokeTestCase{args: wrapArgs(NewCode("f3", "foo.py", []Param{{"a", nil}, {"b", nil}}, 0, fn), 1, 2), want: newTestTuple(1, 2).ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f3", "foo.py", []Param{{"a", nil}, {"b", nil}}, 0, fn), 1), kwargs: wrapKWArgs("b", "bear"), want: newTestTuple(1, "bear").ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f3", "foo.py", []Param{{"a", nil}, {"b", nil}}, 0, fn)), kwargs: wrapKWArgs("b", "bear", "a", "apple"), want: newTestTuple("apple", "bear").ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f3", "foo.py", []Param{{"a", nil}, {"b", nil}}, 0, fn), 1), kwargs: wrapKWArgs("a", "alpha"), wantExc: mustCreateException(TypeErrorType, "f3() got multiple values for keyword argument 'a'")}, invokeTestCase{args: wrapArgs(NewCode("f4", "foo.py", []Param{{"a", nil}, {"b", None}}, 0, fn), 123), want: newTestTuple(123, None).ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f4", "foo.py", []Param{{"a", nil}, {"b", None}}, 0, fn), 123, "bar"), want: newTestTuple(123, "bar").ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f4", "foo.py", []Param{{"a", nil}, {"b", None}}, 0, fn)), kwargs: wrapKWArgs("a", 123, "b", "bar"), want: newTestTuple(123, "bar").ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f5", "foo.py", []Param{{"a", nil}}, CodeFlagVarArg, fn), 1), want: newTestTuple(1, NewTuple()).ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f5", "foo.py", []Param{{"a", nil}}, CodeFlagVarArg, fn), 1, 2, 3), want: newTestTuple(1, newTestTuple(2, 3)).ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f6", "foo.py", []Param{{"a", nil}}, CodeFlagKWArg, fn), "bar"), want: newTestTuple("bar", NewDict()).ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f6", "foo.py", []Param{{"a", nil}}, CodeFlagKWArg, fn)), kwargs: wrapKWArgs("a", "apple", "b", "bear"), want: newTestTuple("apple", newTestDict("b", "bear")).ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f6", "foo.py", []Param{{"a", nil}}, CodeFlagKWArg, fn), "bar"), kwargs: wrapKWArgs("b", "baz", "c", "qux"), want: newTestTuple("bar", newTestDict("b", "baz", "c", "qux")).ToObject()}, invokeTestCase{args: wrapArgs(NewCode("f7", "foo.py", nil, 0, nilFn)), want: None}, } for _, cas := range cases { if err := runInvokeTestCase(testFunc.ToObject(), &cas); err != "" { t.Error(err) } } } func TestCodeEvalRestoreExc(t *testing.T) { e := mustCreateException(RuntimeErrorType, "uh oh") ranC1, ranC2 := false, false globals := NewDict() c1 := NewCode("", "foo.py", nil, 0, func(f *Frame, _ []*Object) (*Object, *BaseException) { if got, _ := f.ExcInfo(); got != e { t.Errorf("ExcInfo() = %v, want %v", got, e) } f.RestoreExc(nil, nil) ranC1 = true return None, nil }) c2 := NewCode("", "foo.py", nil, 0, func(f *Frame, _ []*Object) (*Object, *BaseException) { f.RestoreExc(e, newTraceback(f, nil)) c1.Eval(f, globals, nil, nil) // The exception was cleared by c1 but when returning to c2, it // should have been restored. if got, _ := f.ExcInfo(); got != e { t.Errorf("ExcInfo() = %v, want ", got) } f.RestoreExc(nil, nil) ranC2 = true return None, nil }) c2.Eval(NewRootFrame(), globals, nil, nil) if !ranC1 { t.Error("c1 did not run") } if !ranC2 { t.Error("c2 did not run") } } ================================================ FILE: runtime/complex.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "errors" "fmt" "math" "math/cmplx" "reflect" "regexp" "strconv" "strings" ) // ComplexType is the object representing the Python 'complex' type. var ComplexType = newBasisType("complex", reflect.TypeOf(Complex{}), toComplexUnsafe, ObjectType) // Complex represents Python 'complex' objects. type Complex struct { Object value complex128 } // NewComplex returns a new Complex holding the given complex value. func NewComplex(value complex128) *Complex { return &Complex{Object{typ: ComplexType}, value} } func toComplexUnsafe(o *Object) *Complex { return (*Complex)(o.toPointer()) } // ToObject upcasts c to an Object. func (c *Complex) ToObject() *Object { return &c.Object } // Value returns the underlying complex value held by c. func (c *Complex) Value() complex128 { return c.value } func complexAbs(f *Frame, o *Object) (*Object, *BaseException) { c := toComplexUnsafe(o).Value() return NewFloat(cmplx.Abs(c)).ToObject(), nil } func complexAdd(f *Frame, v, w *Object) (*Object, *BaseException) { return complexArithmeticOp(f, "__add__", v, w, func(lhs, rhs complex128) complex128 { return lhs + rhs }) } func complexCompareNotSupported(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) || w.isInstance(LongType) || w.isInstance(FloatType) || w.isInstance(ComplexType) { return nil, f.RaiseType(TypeErrorType, "no ordering relation is defined for complex numbers") } return NotImplemented, nil } func complexComplex(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func complexDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return complexDivModOp(f, "__div__", v, w, func(v, w complex128) (complex128, bool) { if w == 0 { return 0, false } return v / w, true }) } func complexDivMod(f *Frame, v, w *Object) (*Object, *BaseException) { return complexDivAndModOp(f, "__divmod__", v, w, func(v, w complex128) (complex128, complex128, bool) { if w == 0 { return 0, 0, false } return complexFloorDivOp(v, w), complexModOp(v, w), true }) } func complexEq(f *Frame, v, w *Object) (*Object, *BaseException) { e, ok := complexCompare(toComplexUnsafe(v), w) if !ok { return NotImplemented, nil } return GetBool(e).ToObject(), nil } func complexFloorDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return complexDivModOp(f, "__floordiv__", v, w, func(v, w complex128) (complex128, bool) { if w == 0 { return 0, false } return complexFloorDivOp(v, w), true }) } func complexHash(f *Frame, o *Object) (*Object, *BaseException) { v := toComplexUnsafe(o).Value() hashCombined := hashFloat(real(v)) + 1000003*hashFloat(imag(v)) if hashCombined == -1 { hashCombined = -2 } return NewInt(hashCombined).ToObject(), nil } func complexMod(f *Frame, v, w *Object) (*Object, *BaseException) { return complexDivModOp(f, "__mod__", v, w, func(v, w complex128) (complex128, bool) { if w == 0 { return 0, false } return complexModOp(v, w), true }) } func complexMul(f *Frame, v, w *Object) (*Object, *BaseException) { return complexArithmeticOp(f, "__mul__", v, w, func(lhs, rhs complex128) complex128 { return lhs * rhs }) } func complexNE(f *Frame, v, w *Object) (*Object, *BaseException) { e, ok := complexCompare(toComplexUnsafe(v), w) if !ok { return NotImplemented, nil } return GetBool(!e).ToObject(), nil } func complexNeg(f *Frame, o *Object) (*Object, *BaseException) { c := toComplexUnsafe(o).Value() return NewComplex(-c).ToObject(), nil } func complexNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) if argc == 0 { return newObject(t), nil } if argc > 2 { return nil, f.RaiseType(TypeErrorType, "'__new__' of 'complex' requires at most 2 arguments") } if t != ComplexType { // Allocate a plain complex then copy it's value into an object // of the complex subtype. x, raised := complexNew(f, ComplexType, args, nil) if raised != nil { return nil, raised } result := toComplexUnsafe(newObject(t)) result.value = toComplexUnsafe(x).Value() return result.ToObject(), nil } if complexSlot := args[0].typ.slots.Complex; complexSlot != nil && argc == 1 { c, raised := complexConvert(complexSlot, f, args[0]) if raised != nil { return nil, raised } return c.ToObject(), nil } if args[0].isInstance(StrType) { if argc > 1 { return nil, f.RaiseType(TypeErrorType, "complex() can't take second arg if first is a string") } s := toStrUnsafe(args[0]).Value() result, err := parseComplex(s) if err != nil { return nil, f.RaiseType(ValueErrorType, "complex() arg is a malformed string") } return NewComplex(result).ToObject(), nil } if argc > 1 && args[1].isInstance(StrType) { return nil, f.RaiseType(TypeErrorType, "complex() second arg can't be a string") } cr, raised := complex128Convert(f, args[0]) if raised != nil { return nil, raised } var ci complex128 if argc > 1 { ci, raised = complex128Convert(f, args[1]) if raised != nil { return nil, raised } } // Logically it should be enough to return this: // NewComplex(cr + ci*1i).ToObject() // But Go complex arithmatic is not satisfying all conditions, for instance: // cr := complex(math.Inf(1), 0) // ci := complex(math.Inf(-1), 0) // fmt.Println(cr + ci*1i) // Output is (NaN-Infi), instead of (+Inf-Infi). return NewComplex(complex(real(cr)-imag(ci), imag(cr)+real(ci))).ToObject(), nil } func complexNonZero(f *Frame, o *Object) (*Object, *BaseException) { return GetBool(toComplexUnsafe(o).Value() != 0).ToObject(), nil } func complexPos(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func complexPow(f *Frame, v, w *Object) (*Object, *BaseException) { return complexArithmeticOp(f, "__pow__", v, w, func(lhs, rhs complex128) complex128 { return cmplx.Pow(lhs, rhs) }) } func complexRAdd(f *Frame, v, w *Object) (*Object, *BaseException) { return complexArithmeticOp(f, "__radd__", v, w, func(lhs, rhs complex128) complex128 { return lhs + rhs }) } func complexRDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return complexDivModOp(f, "__rdiv__", v, w, func(v, w complex128) (complex128, bool) { if v == 0 { return 0, false } return w / v, true }) } func complexRDivMod(f *Frame, v, w *Object) (*Object, *BaseException) { return complexDivAndModOp(f, "__rdivmod__", v, w, func(v, w complex128) (complex128, complex128, bool) { if v == 0 { return 0, 0, false } return complexFloorDivOp(w, v), complexModOp(w, v), true }) } func complexRepr(f *Frame, o *Object) (*Object, *BaseException) { c := toComplexUnsafe(o).Value() rs, is := "", "" pre, post := "", "" sign := "" if real(c) == 0.0 { is = strconv.FormatFloat(imag(c), 'g', -1, 64) } else { pre = "(" rs = strconv.FormatFloat(real(c), 'g', -1, 64) is = strconv.FormatFloat(imag(c), 'g', -1, 64) if imag(c) >= 0.0 || math.IsNaN(imag(c)) { sign = "+" } post = ")" } rs = unsignPositiveInf(strings.ToLower(rs)) is = unsignPositiveInf(strings.ToLower(is)) return NewStr(fmt.Sprintf("%s%s%s%sj%s", pre, rs, sign, is, post)).ToObject(), nil } func complexRFloorDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return complexDivModOp(f, "__rfloordiv__", v, w, func(v, w complex128) (complex128, bool) { if v == 0 { return 0, false } return complexFloorDivOp(w, v), true }) } func complexRMod(f *Frame, v, w *Object) (*Object, *BaseException) { return complexDivModOp(f, "__rmod__", v, w, func(v, w complex128) (complex128, bool) { if v == 0 { return 0, false } return complexModOp(w, v), true }) } func complexRMul(f *Frame, v, w *Object) (*Object, *BaseException) { return complexArithmeticOp(f, "__rmul__", v, w, func(lhs, rhs complex128) complex128 { return rhs * lhs }) } func complexRPow(f *Frame, v, w *Object) (*Object, *BaseException) { return complexArithmeticOp(f, "__rpow__", v, w, func(lhs, rhs complex128) complex128 { return cmplx.Pow(rhs, lhs) }) } func complexRSub(f *Frame, v, w *Object) (*Object, *BaseException) { return complexArithmeticOp(f, "__rsub__", v, w, func(lhs, rhs complex128) complex128 { return rhs - lhs }) } func complexSub(f *Frame, v, w *Object) (*Object, *BaseException) { return complexArithmeticOp(f, "__sub__", v, w, func(lhs, rhs complex128) complex128 { return lhs - rhs }) } func initComplexType(dict map[string]*Object) { ComplexType.slots.Abs = &unaryOpSlot{complexAbs} ComplexType.slots.Add = &binaryOpSlot{complexAdd} ComplexType.slots.Complex = &unaryOpSlot{complexComplex} ComplexType.slots.Div = &binaryOpSlot{complexDiv} ComplexType.slots.DivMod = &binaryOpSlot{complexDivMod} ComplexType.slots.Eq = &binaryOpSlot{complexEq} ComplexType.slots.FloorDiv = &binaryOpSlot{complexFloorDiv} ComplexType.slots.GE = &binaryOpSlot{complexCompareNotSupported} ComplexType.slots.GT = &binaryOpSlot{complexCompareNotSupported} ComplexType.slots.Hash = &unaryOpSlot{complexHash} ComplexType.slots.LE = &binaryOpSlot{complexCompareNotSupported} ComplexType.slots.LT = &binaryOpSlot{complexCompareNotSupported} ComplexType.slots.Mod = &binaryOpSlot{complexMod} ComplexType.slots.Mul = &binaryOpSlot{complexMul} ComplexType.slots.NE = &binaryOpSlot{complexNE} ComplexType.slots.Neg = &unaryOpSlot{complexNeg} ComplexType.slots.New = &newSlot{complexNew} ComplexType.slots.NonZero = &unaryOpSlot{complexNonZero} ComplexType.slots.Pos = &unaryOpSlot{complexPos} ComplexType.slots.Pow = &binaryOpSlot{complexPow} ComplexType.slots.RAdd = &binaryOpSlot{complexRAdd} ComplexType.slots.RDiv = &binaryOpSlot{complexRDiv} ComplexType.slots.RDivMod = &binaryOpSlot{complexRDivMod} ComplexType.slots.RFloorDiv = &binaryOpSlot{complexRFloorDiv} ComplexType.slots.Repr = &unaryOpSlot{complexRepr} ComplexType.slots.RMod = &binaryOpSlot{complexRMod} ComplexType.slots.RMul = &binaryOpSlot{complexRMul} ComplexType.slots.RPow = &binaryOpSlot{complexRPow} ComplexType.slots.RSub = &binaryOpSlot{complexRSub} ComplexType.slots.Sub = &binaryOpSlot{complexSub} } func complex128Convert(f *Frame, o *Object) (complex128, *BaseException) { if complexSlot := o.typ.slots.Complex; complexSlot != nil { c, raised := complexConvert(complexSlot, f, o) if raised != nil { return complex(0, 0), raised } return c.Value(), nil } else if floatSlot := o.typ.slots.Float; floatSlot != nil { result, raised := floatConvert(floatSlot, f, o) if raised != nil { return complex(0, 0), raised } return complex(result.Value(), 0), nil } else { return complex(0, 0), f.RaiseType(TypeErrorType, "complex() argument must be a string or a number") } } func complexArithmeticOp(f *Frame, method string, v, w *Object, fun func(v, w complex128) complex128) (*Object, *BaseException) { if w.isInstance(ComplexType) { return NewComplex(fun(toComplexUnsafe(v).Value(), toComplexUnsafe(w).Value())).ToObject(), nil } floatW, ok := floatCoerce(w) if !ok { if math.IsInf(floatW, 0) { return nil, f.RaiseType(OverflowErrorType, "long int too large to convert to float") } return NotImplemented, nil } return NewComplex(fun(toComplexUnsafe(v).Value(), complex(floatW, 0))).ToObject(), nil } // complexCoerce will coerce any numeric type to a complex. If all is // well, it will return the complex128 value, and true (OK). If an overflow // occurs, it will return either (+Inf, false) or (-Inf, false) depending // on whether the source value was too large or too small. Note that if the // source number is an infinite float, the result will be infinite without // overflow, (+-Inf, true). // If the input is not a number, it will return (0, false). func complexCoerce(o *Object) (complex128, bool) { if o.isInstance(ComplexType) { return toComplexUnsafe(o).Value(), true } floatO, ok := floatCoerce(o) if !ok { if math.IsInf(floatO, 0) { return complex(floatO, 0.0), false } return 0, false } return complex(floatO, 0.0), true } func complexCompare(v *Complex, w *Object) (bool, bool) { lhsr := real(v.Value()) rhs, ok := complexCoerce(w) if !ok { return false, false } return lhsr == real(rhs) && imag(v.Value()) == imag(rhs), true } func complexConvert(complexSlot *unaryOpSlot, f *Frame, o *Object) (*Complex, *BaseException) { result, raised := complexSlot.Fn(f, o) if raised != nil { return nil, raised } if !result.isInstance(ComplexType) { exc := fmt.Sprintf("__complex__ returned non-complex (type %s)", result.typ.Name()) return nil, f.RaiseType(TypeErrorType, exc) } return toComplexUnsafe(result), nil } func complexDivModOp(f *Frame, method string, v, w *Object, fun func(v, w complex128) (complex128, bool)) (*Object, *BaseException) { complexW, ok := complexCoerce(w) if !ok { if cmplx.IsInf(complexW) { return nil, f.RaiseType(OverflowErrorType, "long int too large to convert to complex") } return NotImplemented, nil } x, ok := fun(toComplexUnsafe(v).Value(), complexW) if !ok { return nil, f.RaiseType(ZeroDivisionErrorType, "complex division or modulo by zero") } return NewComplex(x).ToObject(), nil } func complexDivAndModOp(f *Frame, method string, v, w *Object, fun func(v, w complex128) (complex128, complex128, bool)) (*Object, *BaseException) { complexW, ok := complexCoerce(w) if !ok { if cmplx.IsInf(complexW) { return nil, f.RaiseType(OverflowErrorType, "long int too large to convert to complex") } return NotImplemented, nil } q, m, ok := fun(toComplexUnsafe(v).Value(), complexW) if !ok { return nil, f.RaiseType(ZeroDivisionErrorType, "complex division or modulo by zero") } return NewTuple2(NewComplex(q).ToObject(), NewComplex(m).ToObject()).ToObject(), nil } func complexFloorDivOp(v, w complex128) complex128 { return complex(math.Floor(real(v/w)), 0) } func complexModOp(v, w complex128) complex128 { return v - complexFloorDivOp(v, w)*w } const ( blank = iota real1 imag1 real2 sign2 imag3 real4 sign5 onlyJ ) // ParseComplex converts the string s to a complex number. // If string is well-formed (one of these forms: , j, // j, j, j or j, where is // any numeric string that's acceptable by strconv.ParseFloat(s, 64)), // ParseComplex returns the respective complex128 number. func parseComplex(s string) (complex128, error) { c := strings.Count(s, "(") if (c > 1) || (c == 1 && strings.Count(s, ")") != 1) { return complex(0, 0), errors.New("Malformed complex string, more than one matching parantheses") } ts := strings.TrimSpace(s) ts = strings.Trim(ts, "()") ts = strings.TrimSpace(ts) re := `(?i)(?:(?:(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?)|(?:infinity)|(?:nan)|(?:inf))` fre := `[-+]?` + re sre := `[-+]` + re fsfj := `(?:(?P` + fre + `)(?P` + sre + `)j)` fsj := `(?:(?P` + fre + `)(?P[-+])j)` fj := `(?P` + fre + `)j` f := `(?P` + fre + `)` sj := `(?P[-+])j` j := `(?Pj)` r := regexp.MustCompile(`^(?:` + fsfj + `|` + fsj + `|` + fj + `|` + f + `|` + sj + `|` + j + `)$`) subs := r.FindStringSubmatch(ts) if subs == nil { return complex(0, 0), errors.New("Malformed complex string, no mathing pattern found") } if subs[real1] != "" && subs[imag1] != "" { r, _ := strconv.ParseFloat(unsignNaN(subs[real1]), 64) i, err := strconv.ParseFloat(unsignNaN(subs[imag1]), 64) return complex(r, i), err } if subs[real2] != "" && subs[sign2] != "" { r, err := strconv.ParseFloat(unsignNaN(subs[real2]), 64) if subs[sign2] == "-" { return complex(r, -1), err } return complex(r, 1), err } if subs[imag3] != "" { i, err := strconv.ParseFloat(unsignNaN(subs[imag3]), 64) return complex(0, i), err } if subs[real4] != "" { r, err := strconv.ParseFloat(unsignNaN(subs[real4]), 64) return complex(r, 0), err } if subs[sign5] != "" { if subs[sign5] == "-" { return complex(0, -1), nil } return complex(0, 1), nil } if subs[onlyJ] != "" { return complex(0, 1), nil } return complex(0, 0), errors.New("Malformed complex string") } func unsignNaN(s string) string { ls := strings.ToLower(s) if ls == "-nan" || ls == "+nan" { return "nan" } return s } ================================================ FILE: runtime/complex_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "errors" "math" "math/big" "math/cmplx" "testing" ) func TestComplexAbs(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(complex(0, 0)), want: NewFloat(0).ToObject()}, {args: wrapArgs(complex(1, 1)), want: NewFloat(1.4142135623730951).ToObject()}, {args: wrapArgs(complex(1, 2)), want: NewFloat(2.23606797749979).ToObject()}, {args: wrapArgs(complex(3, 4)), want: NewFloat(5).ToObject()}, {args: wrapArgs(complex(-3, 4)), want: NewFloat(5).ToObject()}, {args: wrapArgs(complex(3, -4)), want: NewFloat(5).ToObject()}, {args: wrapArgs(-complex(3, 4)), want: NewFloat(5).ToObject()}, {args: wrapArgs(complex(0.123456e-3, 0)), want: NewFloat(0.000123456).ToObject()}, {args: wrapArgs(complex(0.123456e-3, 3.14151692e+7)), want: NewFloat(31415169.2).ToObject()}, {args: wrapArgs(complex(math.Inf(-1), 1.2)), want: NewFloat(math.Inf(1)).ToObject()}, {args: wrapArgs(complex(3.4, math.Inf(1))), want: NewFloat(math.Inf(1)).ToObject()}, {args: wrapArgs(complex(math.Inf(1), math.Inf(-1))), want: NewFloat(math.Inf(1)).ToObject()}, {args: wrapArgs(complex(math.Inf(1), math.NaN())), want: NewFloat(math.Inf(1)).ToObject()}, {args: wrapArgs(complex(math.NaN(), math.Inf(1))), want: NewFloat(math.Inf(1)).ToObject()}, {args: wrapArgs(complex(math.NaN(), 5.6)), want: NewFloat(math.NaN()).ToObject()}, {args: wrapArgs(complex(7.8, math.NaN())), want: NewFloat(math.NaN()).ToObject()}, } for _, cas := range cases { switch got, match := checkInvokeResult(wrapFuncForTest(complexAbs), cas.args, cas.want, cas.wantExc); match { case checkInvokeResultReturnValueMismatch: if got == nil || cas.want == nil || !got.isInstance(FloatType) || !cas.want.isInstance(FloatType) || !floatsAreSame(toFloatUnsafe(got).Value(), toFloatUnsafe(cas.want).Value()) { t.Errorf("complex.__abs__%v = %v, want %v", cas.args, got, cas.want) } } } } func TestComplexEq(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(complex(0, 0), 0), want: True.ToObject()}, {args: wrapArgs(complex(1, 0), 0), want: False.ToObject()}, {args: wrapArgs(complex(-12, 0), -12), want: True.ToObject()}, {args: wrapArgs(complex(-12, 0), 1), want: False.ToObject()}, {args: wrapArgs(complex(17.20, 0), 17.20), want: True.ToObject()}, {args: wrapArgs(complex(1.2, 0), 17.20), want: False.ToObject()}, {args: wrapArgs(complex(-4, 15), complex(-4, 15)), want: True.ToObject()}, {args: wrapArgs(complex(-4, 15), complex(1, 2)), want: False.ToObject()}, {args: wrapArgs(complex(math.Inf(1), 0), complex(math.Inf(1), 0)), want: True.ToObject()}, {args: wrapArgs(complex(math.Inf(1), 0), complex(0, math.Inf(1))), want: False.ToObject()}, {args: wrapArgs(complex(math.Inf(-1), 0), complex(math.Inf(-1), 0)), want: True.ToObject()}, {args: wrapArgs(complex(math.Inf(-1), 0), complex(0, math.Inf(-1))), want: False.ToObject()}, {args: wrapArgs(complex(math.Inf(1), math.Inf(1)), complex(math.Inf(1), math.Inf(1))), want: True.ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(complexEq), &cas); err != "" { t.Error(err) } } } // FIXME(corona10): Since Go 1.9 moved to C99 float division and what CPython uses as well. // Some tests can be failed with version < Go 1.9. We need to detect Go version. // And changed expected values. func TestComplexBinaryOps(t *testing.T) { cases := []struct { fun func(f *Frame, v, w *Object) (*Object, *BaseException) v, w *Object want *Object wantExc *BaseException }{ {Add, NewComplex(1 + 3i).ToObject(), NewInt(1).ToObject(), NewComplex(2 + 3i).ToObject(), nil}, {Add, NewComplex(1 + 3i).ToObject(), NewFloat(-1).ToObject(), NewComplex(3i).ToObject(), nil}, {Add, NewComplex(1 + 3i).ToObject(), NewInt(1).ToObject(), NewComplex(2 + 3i).ToObject(), nil}, {Add, NewComplex(1 + 3i).ToObject(), NewComplex(-1 - 3i).ToObject(), NewComplex(0i).ToObject(), nil}, {Add, NewFloat(math.Inf(1)).ToObject(), NewComplex(3i).ToObject(), NewComplex(complex(math.Inf(1), 3)).ToObject(), nil}, {Add, NewFloat(math.Inf(-1)).ToObject(), NewComplex(3i).ToObject(), NewComplex(complex(math.Inf(-1), 3)).ToObject(), nil}, {Add, NewFloat(math.NaN()).ToObject(), NewComplex(3i).ToObject(), NewComplex(complex(math.NaN(), 3)).ToObject(), nil}, {Add, NewComplex(cmplx.NaN()).ToObject(), NewComplex(3i).ToObject(), NewComplex(cmplx.NaN()).ToObject(), nil}, {Add, NewFloat(math.Inf(-1)).ToObject(), NewComplex(complex(math.Inf(+1), 3)).ToObject(), NewComplex(complex(math.NaN(), 3)).ToObject(), nil}, {Add, NewComplex(1 + 3i).ToObject(), None, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'complex' and 'NoneType'")}, {Add, None, NewComplex(1 + 3i).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'NoneType' and 'complex'")}, {Add, NewInt(3).ToObject(), NewComplex(3i).ToObject(), NewComplex(3 + 3i).ToObject(), nil}, {Add, NewLong(big.NewInt(9999999)).ToObject(), NewComplex(3i).ToObject(), NewComplex(9999999 + 3i).ToObject(), nil}, {Add, NewFloat(3.5).ToObject(), NewComplex(3i).ToObject(), NewComplex(3.5 + 3i).ToObject(), nil}, {Div, NewComplex(1 + 2i).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(1 + 0i).ToObject(), nil}, {Div, NewComplex(3 + 4i).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(2.2 - 0.4i).ToObject(), nil}, {Div, NewComplex(3.14 - 0.618i).ToObject(), NewComplex(-0.123e-4 + 0.151692i).ToObject(), NewComplex(-4.075723201992163 - 20.69950866627519i).ToObject(), nil}, {Div, NewInt(3).ToObject(), NewComplex(3 - 4i).ToObject(), NewComplex(0.36 + 0.48i).ToObject(), nil}, {Div, NewComplex(3 + 4i).ToObject(), NewInt(-5).ToObject(), NewComplex(-0.6 - 0.8i).ToObject(), nil}, {Div, NewFloat(1.2).ToObject(), NewComplex(1 - 2i).ToObject(), NewComplex(0.24 + 0.48i).ToObject(), nil}, {Div, NewComplex(1 + 2i).ToObject(), NewFloat(-3.4).ToObject(), NewComplex(-0.29411764705882354 - 0.5882352941176471i).ToObject(), nil}, {Div, NewLong(big.NewInt(123)).ToObject(), NewComplex(3 + 4i).ToObject(), NewComplex(14.76 - 19.68i).ToObject(), nil}, {Div, NewComplex(3 - 4i).ToObject(), NewLong(big.NewInt(-34)).ToObject(), NewComplex(-0.08823529411764706 + 0.11764705882352941i).ToObject(), nil}, {Div, NewComplex(3 + 4i).ToObject(), NewComplex(complex(math.Inf(1), math.Inf(-1))).ToObject(), NewComplex(0i).ToObject(), nil}, {Div, NewComplex(3 + 4i).ToObject(), NewComplex(complex(math.Inf(1), 2)).ToObject(), NewComplex(0i).ToObject(), nil}, {Div, NewComplex(complex(math.Inf(1), math.Inf(1))).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.Inf(1), math.NaN())).ToObject(), nil}, {Div, NewComplex(complex(math.Inf(1), 4)).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.Inf(1), math.Inf(-1))).ToObject(), nil}, {Div, NewComplex(complex(3, math.Inf(1))).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.Inf(1), math.Inf(1))).ToObject(), nil}, {Div, NewComplex(complex(3, math.NaN())).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject(), nil}, {Div, NewStr("foo").ToObject(), NewComplex(1 + 2i).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for /: 'str' and 'complex'")}, {Div, NewComplex(3 + 4i).ToObject(), NewComplex(0 + 0i).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "complex division or modulo by zero")}, {Div, NewComplex(complex(math.Inf(1), math.NaN())).ToObject(), NewComplex(0 + 0i).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "complex division or modulo by zero")}, {Div, NewComplex(3 + 4i).ToObject(), NewLong(bigLongNumber).ToObject(), nil, mustCreateException(OverflowErrorType, "long int too large to convert to complex")}, {FloorDiv, NewComplex(1 + 2i).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(1 + 0i).ToObject(), nil}, {FloorDiv, NewComplex(3 + 4i).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(2 - 0i).ToObject(), nil}, {FloorDiv, NewComplex(3.14 - 0.618i).ToObject(), NewComplex(-0.123e-4 + 0.151692i).ToObject(), NewComplex(-5 - 0i).ToObject(), nil}, {FloorDiv, NewInt(3).ToObject(), NewComplex(3 - 4i).ToObject(), NewComplex(0i).ToObject(), nil}, {FloorDiv, NewComplex(3 + 4i).ToObject(), NewInt(-5).ToObject(), NewComplex(-1 + 0i).ToObject(), nil}, {FloorDiv, NewFloat(1.2).ToObject(), NewComplex(1 - 2i).ToObject(), NewComplex(0i).ToObject(), nil}, {FloorDiv, NewComplex(1 + 2i).ToObject(), NewFloat(-3.4).ToObject(), NewComplex(-1 + 0i).ToObject(), nil}, {FloorDiv, NewLong(big.NewInt(123)).ToObject(), NewComplex(3 + 4i).ToObject(), NewComplex(14 - 0i).ToObject(), nil}, {FloorDiv, NewComplex(3 - 4i).ToObject(), NewLong(big.NewInt(-34)).ToObject(), NewComplex(-1 + 0i).ToObject(), nil}, {FloorDiv, NewComplex(3 + 4i).ToObject(), NewComplex(complex(math.Inf(1), math.Inf(-1))).ToObject(), NewComplex(0i).ToObject(), nil}, {FloorDiv, NewComplex(3 + 4i).ToObject(), NewComplex(complex(math.Inf(1), 2)).ToObject(), NewComplex(0i).ToObject(), nil}, {FloorDiv, NewComplex(complex(math.Inf(1), math.Inf(1))).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.Inf(1), 0)).ToObject(), nil}, {FloorDiv, NewComplex(complex(math.Inf(1), 4)).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.Inf(1), 0)).ToObject(), nil}, {FloorDiv, NewComplex(complex(3, math.Inf(1))).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.Inf(1), 0)).ToObject(), nil}, {FloorDiv, NewComplex(complex(3, math.NaN())).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.NaN(), 0)).ToObject(), nil}, {FloorDiv, NewStr("foo").ToObject(), NewComplex(1 + 2i).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for //: 'str' and 'complex'")}, {FloorDiv, NewComplex(3 + 4i).ToObject(), NewComplex(0 + 0i).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "complex division or modulo by zero")}, {FloorDiv, NewComplex(complex(math.Inf(1), math.NaN())).ToObject(), NewComplex(0 + 0i).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "complex division or modulo by zero")}, {FloorDiv, NewComplex(3 + 4i).ToObject(), NewLong(bigLongNumber).ToObject(), nil, mustCreateException(OverflowErrorType, "long int too large to convert to complex")}, {Mod, NewComplex(3 + 4i).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(1 + 0i).ToObject(), nil}, {Mod, NewComplex(1 + 2i).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(0i).ToObject(), nil}, {Mod, NewComplex(3.14 - 0.618i).ToObject(), NewComplex(-0.123e-4 + 0.151692i).ToObject(), NewComplex(3.1399385 + 0.14045999999999992i).ToObject(), nil}, {Mod, NewInt(3).ToObject(), NewComplex(3 - 4i).ToObject(), NewComplex(3 + 0i).ToObject(), nil}, {Mod, NewComplex(3 + 4i).ToObject(), NewInt(-5).ToObject(), NewComplex(-2 + 4i).ToObject(), nil}, {Mod, NewFloat(1.2).ToObject(), NewComplex(1 - 2i).ToObject(), NewComplex(1.2 + 0i).ToObject(), nil}, {Mod, NewComplex(1 + 2i).ToObject(), NewFloat(-3.4).ToObject(), NewComplex(-2.4 + 2i).ToObject(), nil}, {Mod, NewLong(big.NewInt(123)).ToObject(), NewComplex(3 + 4i).ToObject(), NewComplex(81 - 56i).ToObject(), nil}, {Mod, NewComplex(3 - 4i).ToObject(), NewLong(big.NewInt(-34)).ToObject(), NewComplex(-31 - 4i).ToObject(), nil}, {Mod, NewComplex(3 + 4i).ToObject(), NewComplex(complex(math.Inf(1), math.Inf(-1))).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject(), nil}, {Mod, NewComplex(3 + 4i).ToObject(), NewComplex(complex(math.Inf(1), 2)).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject(), nil}, {Mod, NewComplex(complex(math.Inf(1), math.Inf(1))).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject(), nil}, {Mod, NewComplex(complex(math.Inf(1), 4)).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.NaN(), math.Inf(-1))).ToObject(), nil}, {Mod, NewComplex(complex(3, math.Inf(1))).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.Inf(-1), math.NaN())).ToObject(), nil}, {Mod, NewComplex(complex(3, math.NaN())).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject(), nil}, {Mod, NewStr("foo").ToObject(), NewComplex(1 + 2i).ToObject(), nil, mustCreateException(TypeErrorType, "not all arguments converted during string formatting")}, {Mod, NewComplex(3 + 4i).ToObject(), NewComplex(0 + 0i).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "complex division or modulo by zero")}, {Mod, NewComplex(complex(math.Inf(1), math.NaN())).ToObject(), NewComplex(0 + 0i).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "complex division or modulo by zero")}, {Mod, NewComplex(3 + 4i).ToObject(), NewLong(bigLongNumber).ToObject(), nil, mustCreateException(OverflowErrorType, "long int too large to convert to complex")}, {Sub, NewComplex(1 + 3i).ToObject(), NewComplex(1 + 3i).ToObject(), NewComplex(0i).ToObject(), nil}, {Sub, NewComplex(1 + 3i).ToObject(), NewComplex(3i).ToObject(), NewComplex(1).ToObject(), nil}, {Sub, NewComplex(1 + 3i).ToObject(), NewFloat(1).ToObject(), NewComplex(3i).ToObject(), nil}, {Sub, NewComplex(3i).ToObject(), NewFloat(1.2).ToObject(), NewComplex(-1.2 + 3i).ToObject(), nil}, {Sub, NewComplex(1 + 3i).ToObject(), NewComplex(1 + 3i).ToObject(), NewComplex(0i).ToObject(), nil}, {Sub, NewComplex(4 + 3i).ToObject(), NewInt(1).ToObject(), NewComplex(3 + 3i).ToObject(), nil}, {Sub, NewComplex(4 + 3i).ToObject(), NewLong(big.NewInt(99994)).ToObject(), NewComplex(-99990 + 3i).ToObject(), nil}, {Sub, NewFloat(math.Inf(1)).ToObject(), NewComplex(3i).ToObject(), NewComplex(complex(math.Inf(1), -3)).ToObject(), nil}, {Sub, NewFloat(math.Inf(-1)).ToObject(), NewComplex(3i).ToObject(), NewComplex(complex(math.Inf(-1), -3)).ToObject(), nil}, {Sub, NewComplex(1 + 3i).ToObject(), None, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for -: 'complex' and 'NoneType'")}, {Sub, None, NewComplex(1 + 3i).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for -: 'NoneType' and 'complex'")}, {Sub, NewFloat(math.NaN()).ToObject(), NewComplex(3i).ToObject(), NewComplex(complex(math.NaN(), -3)).ToObject(), nil}, {Sub, NewComplex(cmplx.NaN()).ToObject(), NewComplex(3i).ToObject(), NewComplex(cmplx.NaN()).ToObject(), nil}, {Sub, NewFloat(math.Inf(-1)).ToObject(), NewComplex(complex(math.Inf(-1), 3)).ToObject(), NewComplex(complex(math.NaN(), -3)).ToObject(), nil}, {Mul, NewComplex(1 + 3i).ToObject(), NewComplex(1 + 3i).ToObject(), NewComplex(-8 + 6i).ToObject(), nil}, {Mul, NewComplex(1 + 3i).ToObject(), NewComplex(3i).ToObject(), NewComplex(-9 + 3i).ToObject(), nil}, {Mul, NewComplex(1 + 3i).ToObject(), NewFloat(1).ToObject(), NewComplex(1 + 3i).ToObject(), nil}, {Mul, NewComplex(3i).ToObject(), NewFloat(1.2).ToObject(), NewComplex(3.5999999999999996i).ToObject(), nil}, {Mul, NewComplex(1 + 3i).ToObject(), NewComplex(1 + 3i).ToObject(), NewComplex(-8 + 6i).ToObject(), nil}, {Mul, NewComplex(4 + 3i).ToObject(), NewInt(1).ToObject(), NewComplex(4 + 3i).ToObject(), nil}, {Mul, NewComplex(4 + 3i).ToObject(), NewLong(big.NewInt(99994)).ToObject(), NewComplex(399976 + 299982i).ToObject(), nil}, {Mul, NewFloat(math.Inf(1)).ToObject(), NewComplex(3i).ToObject(), NewComplex(complex(math.NaN(), math.Inf(1))).ToObject(), nil}, {Mul, NewFloat(math.Inf(-1)).ToObject(), NewComplex(3i).ToObject(), NewComplex(complex(math.NaN(), math.Inf(-1))).ToObject(), nil}, {Mul, NewComplex(1 + 3i).ToObject(), None, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'complex' and 'NoneType'")}, {Mul, None, NewComplex(1 + 3i).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'NoneType' and 'complex'")}, {Mul, NewFloat(math.NaN()).ToObject(), NewComplex(3i).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject(), nil}, {Mul, NewComplex(cmplx.NaN()).ToObject(), NewComplex(3i).ToObject(), NewComplex(cmplx.NaN()).ToObject(), nil}, {Mul, NewFloat(math.Inf(-1)).ToObject(), NewComplex(complex(math.Inf(-1), 3)).ToObject(), NewComplex(complex(math.Inf(1), math.NaN())).ToObject(), nil}, {Pow, NewComplex(0i).ToObject(), NewComplex(0i).ToObject(), NewComplex(1 + 0i).ToObject(), nil}, {Pow, NewComplex(-1 + 0i).ToObject(), NewComplex(1i).ToObject(), NewComplex(0.04321391826377226 + 0i).ToObject(), nil}, {Pow, NewComplex(1 + 2i).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(-0.22251715680177264 + 0.10070913113607538i).ToObject(), nil}, {Pow, NewComplex(0i).ToObject(), NewComplex(-1 + 0i).ToObject(), NewComplex(complex(math.Inf(1), 0)).ToObject(), nil}, {Pow, NewComplex(0i).ToObject(), NewComplex(-1 + 1i).ToObject(), NewComplex(complex(math.Inf(1), math.Inf(1))).ToObject(), nil}, {Pow, NewComplex(complex(math.Inf(-1), 2)).ToObject(), NewComplex(1 + 2i).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject(), nil}, {Pow, NewComplex(1 + 2i).ToObject(), NewComplex(complex(1, math.Inf(1))).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject(), nil}, {Pow, NewComplex(complex(math.NaN(), 1)).ToObject(), NewComplex(3 + 4i).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject(), nil}, {Pow, NewComplex(3 + 4i).ToObject(), NewInt(3).ToObject(), NewComplex(-117 + 44.00000000000003i).ToObject(), nil}, {Pow, NewComplex(3 + 4i).ToObject(), NewFloat(3.1415).ToObject(), NewComplex(-152.8892667678244 + 35.555335130496516i).ToObject(), nil}, {Pow, NewComplex(3 + 4i).ToObject(), NewLong(big.NewInt(123)).ToObject(), NewComplex(5.393538720276193e+85 + 7.703512580443326e+85i).ToObject(), nil}, {Pow, NewComplex(1 + 2i).ToObject(), NewStr("foo").ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for **: 'complex' and 'str'")}, {Pow, NewStr("foo").ToObject(), NewComplex(1 + 2i).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for **: 'str' and 'complex'")}, } for _, cas := range cases { switch got, result := checkInvokeResult(wrapFuncForTest(cas.fun), []*Object{cas.v, cas.w}, cas.want, cas.wantExc); result { case checkInvokeResultExceptionMismatch: t.Errorf("%s(%v, %v) raised %v, want %v", getFuncName(cas.fun), cas.v, cas.w, got, cas.wantExc) case checkInvokeResultReturnValueMismatch: if got == nil || cas.want == nil || !got.isInstance(ComplexType) || !cas.want.isInstance(ComplexType) || !complexesAreSame(toComplexUnsafe(got).Value(), toComplexUnsafe(cas.want).Value()) { t.Errorf("%s(%v, %v) = %v, want %v", getFuncName(cas.fun), cas.v, cas.w, got, cas.want) } } } } func TestComplexCompareNotSupported(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(complex(1, 2), 1), wantExc: mustCreateException(TypeErrorType, "no ordering relation is defined for complex numbers")}, {args: wrapArgs(complex(1, 2), 1.2), wantExc: mustCreateException(TypeErrorType, "no ordering relation is defined for complex numbers")}, {args: wrapArgs(complex(1, 2), math.NaN()), wantExc: mustCreateException(TypeErrorType, "no ordering relation is defined for complex numbers")}, {args: wrapArgs(complex(1, 2), math.Inf(-1)), wantExc: mustCreateException(TypeErrorType, "no ordering relation is defined for complex numbers")}, {args: wrapArgs(complex(1, 2), "abc"), want: NotImplemented}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(complexCompareNotSupported), &cas); err != "" { t.Error(err) } } } func TestComplexDivMod(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs((1 + 2i), (1 + 2i)), want: NewTuple2(NewComplex(1+0i).ToObject(), NewComplex(0i).ToObject()).ToObject()}, {args: wrapArgs((3 + 4i), (1 + 2i)), want: NewTuple2(NewComplex(2-0i).ToObject(), NewComplex(1+0i).ToObject()).ToObject()}, {args: wrapArgs((3.14 - 0.618i), (-0.123e-4 + 0.151692i)), want: NewTuple2(NewComplex(-5-0i).ToObject(), NewComplex(3.1399385+0.14045999999999992i).ToObject()).ToObject()}, {args: wrapArgs(3, (3 - 4i)), want: NewTuple2(NewComplex(0i).ToObject(), NewComplex(3+0i).ToObject()).ToObject()}, {args: wrapArgs((3 + 4i), -5), want: NewTuple2(NewComplex(-1+0i).ToObject(), NewComplex(-2+4i).ToObject()).ToObject()}, {args: wrapArgs(1.2, (1 - 2i)), want: NewTuple2(NewComplex(0i).ToObject(), NewComplex(1.2+0i).ToObject()).ToObject()}, {args: wrapArgs((1 + 2i), -3.4), want: NewTuple2(NewComplex(-1+0i).ToObject(), NewComplex(-2.4+2i).ToObject()).ToObject()}, {args: wrapArgs(big.NewInt(123), (3 + 4i)), want: NewTuple2(NewComplex(14-0i).ToObject(), NewComplex(81-56i).ToObject()).ToObject()}, {args: wrapArgs((3 - 4i), big.NewInt(-34)), want: NewTuple2(NewComplex(-1+0i).ToObject(), NewComplex(-31-4i).ToObject()).ToObject()}, {args: wrapArgs((3 + 4i), complex(math.Inf(1), math.Inf(-1))), want: NewTuple2(NewComplex(0i).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject()).ToObject()}, {args: wrapArgs((3 + 4i), complex(math.Inf(1), 2)), want: NewTuple2(NewComplex(0i).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject()).ToObject()}, {args: wrapArgs(complex(math.Inf(1), math.Inf(1)), (1 + 2i)), want: NewTuple2(NewComplex(complex(math.Inf(1), 0)).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject()).ToObject()}, {args: wrapArgs(complex(math.Inf(1), 4), (1 + 2i)), want: NewTuple2(NewComplex(complex(math.Inf(1), 0)).ToObject(), NewComplex(complex(math.NaN(), math.Inf(-1))).ToObject()).ToObject()}, {args: wrapArgs(complex(3, math.Inf(1)), (1 + 2i)), want: NewTuple2(NewComplex(complex(math.Inf(1), 0)).ToObject(), NewComplex(complex(math.Inf(-1), math.NaN())).ToObject()).ToObject()}, {args: wrapArgs(complex(3, math.NaN()), (1 + 2i)), want: NewTuple2(NewComplex(complex(math.NaN(), 0)).ToObject(), NewComplex(complex(math.NaN(), math.NaN())).ToObject()).ToObject()}, {args: wrapArgs("foo", (1 + 2i)), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for divmod(): 'str' and 'complex'")}, {args: wrapArgs((3 + 4i), (0 + 0i)), wantExc: mustCreateException(ZeroDivisionErrorType, "complex division or modulo by zero")}, {args: wrapArgs(complex(math.Inf(1), math.NaN()), (0 + 0i)), wantExc: mustCreateException(ZeroDivisionErrorType, "complex division or modulo by zero")}, {args: wrapArgs((3 + 4i), bigLongNumber), wantExc: mustCreateException(OverflowErrorType, "long int too large to convert to complex")}, } for _, cas := range cases { switch got, result := checkInvokeResult(wrapFuncForTest(DivMod), cas.args, cas.want, cas.wantExc); result { case checkInvokeResultExceptionMismatch: t.Errorf("complex.__divmod__%v raised %v, want %v", cas.args, got, cas.wantExc) case checkInvokeResultReturnValueMismatch: // Handle NaN specially, since NaN != NaN. if got == nil || cas.want == nil || !got.isInstance(TupleType) || !cas.want.isInstance(TupleType) || !tupleComplexesAreSame(got, cas.want) { t.Errorf("complex.__divmod__%v = %v, want %v", cas.args, got, cas.want) } } } } func TestComplexNE(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(complex(0, 0), 0), want: False.ToObject()}, {args: wrapArgs(complex(1, 0), 0), want: True.ToObject()}, {args: wrapArgs(complex(-12, 0), -12), want: False.ToObject()}, {args: wrapArgs(complex(-12, 0), 1), want: True.ToObject()}, {args: wrapArgs(complex(17.20, 0), 17.20), want: False.ToObject()}, {args: wrapArgs(complex(1.2, 0), 17.20), want: True.ToObject()}, {args: wrapArgs(complex(-4, 15), complex(-4, 15)), want: False.ToObject()}, {args: wrapArgs(complex(-4, 15), complex(1, 2)), want: True.ToObject()}, {args: wrapArgs(complex(math.Inf(1), 0), complex(math.Inf(1), 0)), want: False.ToObject()}, {args: wrapArgs(complex(math.Inf(1), 0), complex(0, math.Inf(1))), want: True.ToObject()}, {args: wrapArgs(complex(math.Inf(-1), 0), complex(math.Inf(-1), 0)), want: False.ToObject()}, {args: wrapArgs(complex(math.Inf(-1), 0), complex(0, math.Inf(-1))), want: True.ToObject()}, {args: wrapArgs(complex(math.Inf(1), math.Inf(1)), complex(math.Inf(1), math.Inf(1))), want: False.ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(complexNE), &cas); err != "" { t.Error(err) } } } func TestComplexNew(t *testing.T) { complexNew := mustNotRaise(GetAttr(NewRootFrame(), ComplexType.ToObject(), NewStr("__new__"), nil)) goodSlotType := newTestClass("GoodSlot", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__complex__": newBuiltinFunction("__complex__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewComplex(complex(1, 2)).ToObject(), nil }).ToObject(), })) badSlotType := newTestClass("BadSlot", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__complex__": newBuiltinFunction("__complex__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return newObject(ObjectType), nil }).ToObject(), })) strictEqType := newTestClassStrictEq("StrictEq", ComplexType) newStrictEq := func(v complex128) *Object { f := Complex{Object: Object{typ: strictEqType}, value: v} return f.ToObject() } subType := newTestClass("SubType", []*Type{ComplexType}, newStringDict(map[string]*Object{})) subTypeObject := (&Complex{Object: Object{typ: subType}, value: 3.14}).ToObject() slotSubTypeType := newTestClass("SlotSubType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__complex__": newBuiltinFunction("__complex__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return subTypeObject, nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(ComplexType), want: NewComplex(0).ToObject()}, {args: wrapArgs(ComplexType, 56), want: NewComplex(complex(56, 0)).ToObject()}, {args: wrapArgs(ComplexType, -12), want: NewComplex(complex(-12, 0)).ToObject()}, {args: wrapArgs(ComplexType, 3.14), want: NewComplex(complex(3.14, 0)).ToObject()}, {args: wrapArgs(ComplexType, -703.4), want: NewComplex(complex(-703.4, 0)).ToObject()}, {args: wrapArgs(ComplexType, math.NaN()), want: NewComplex(complex(math.NaN(), 0)).ToObject()}, {args: wrapArgs(ComplexType, math.Inf(1)), want: NewComplex(complex(math.Inf(1), 0)).ToObject()}, {args: wrapArgs(ComplexType, math.Inf(-1)), want: NewComplex(complex(math.Inf(-1), 0)).ToObject()}, {args: wrapArgs(ComplexType, biggestFloat), want: NewComplex(complex(math.MaxFloat64, 0)).ToObject()}, {args: wrapArgs(ComplexType, new(big.Int).Neg(biggestFloat)), want: NewComplex(complex(-math.MaxFloat64, 0)).ToObject()}, {args: wrapArgs(ComplexType, new(big.Int).Sub(big.NewInt(-1), biggestFloat)), wantExc: mustCreateException(OverflowErrorType, "long int too large to convert to float")}, {args: wrapArgs(ComplexType, new(big.Int).Add(biggestFloat, big.NewInt(1))), wantExc: mustCreateException(OverflowErrorType, "long int too large to convert to float")}, {args: wrapArgs(ComplexType, bigLongNumber), wantExc: mustCreateException(OverflowErrorType, "long int too large to convert to float")}, {args: wrapArgs(ComplexType, complex(1, 2)), want: NewComplex(complex(1, 2)).ToObject()}, {args: wrapArgs(ComplexType, complex(-0.0001e-1, 3.14151692)), want: NewComplex(complex(-0.00001, 3.14151692)).ToObject()}, {args: wrapArgs(ComplexType, "23"), want: NewComplex(complex(23, 0)).ToObject()}, {args: wrapArgs(ComplexType, "-516"), want: NewComplex(complex(-516, 0)).ToObject()}, {args: wrapArgs(ComplexType, "1.003e4"), want: NewComplex(complex(10030, 0)).ToObject()}, {args: wrapArgs(ComplexType, "151.7"), want: NewComplex(complex(151.7, 0)).ToObject()}, {args: wrapArgs(ComplexType, "-74.02"), want: NewComplex(complex(-74.02, 0)).ToObject()}, {args: wrapArgs(ComplexType, "+38.29"), want: NewComplex(complex(38.29, 0)).ToObject()}, {args: wrapArgs(ComplexType, "8j"), want: NewComplex(complex(0, 8)).ToObject()}, {args: wrapArgs(ComplexType, "-17j"), want: NewComplex(complex(0, -17)).ToObject()}, {args: wrapArgs(ComplexType, "7.3j"), want: NewComplex(complex(0, 7.3)).ToObject()}, {args: wrapArgs(ComplexType, "-4.786j"), want: NewComplex(complex(0, -4.786)).ToObject()}, {args: wrapArgs(ComplexType, "+17.59123j"), want: NewComplex(complex(0, 17.59123)).ToObject()}, {args: wrapArgs(ComplexType, "-3.0007e3j"), want: NewComplex(complex(0, -3000.7)).ToObject()}, {args: wrapArgs(ComplexType, "1+2j"), want: NewComplex(complex(1, 2)).ToObject()}, {args: wrapArgs(ComplexType, "3.1415-23j"), want: NewComplex(complex(3.1415, -23)).ToObject()}, {args: wrapArgs(ComplexType, "-23+3.1415j"), want: NewComplex(complex(-23, 3.1415)).ToObject()}, {args: wrapArgs(ComplexType, "+451.2192+384.27j"), want: NewComplex(complex(451.2192, 384.27)).ToObject()}, {args: wrapArgs(ComplexType, "-38.378-283.28j"), want: NewComplex(complex(-38.378, -283.28)).ToObject()}, {args: wrapArgs(ComplexType, "1.76123e2+0.000007e6j"), want: NewComplex(complex(176.123, 7)).ToObject()}, {args: wrapArgs(ComplexType, "-nan+nanj"), want: NewComplex(complex(math.NaN(), math.NaN())).ToObject()}, {args: wrapArgs(ComplexType, "inf-infj"), want: NewComplex(complex(math.Inf(1), math.Inf(-1))).ToObject()}, {args: wrapArgs(ComplexType, 1, 2), want: NewComplex(complex(1, 2)).ToObject()}, {args: wrapArgs(ComplexType, 7, 23.45), want: NewComplex(complex(7, 23.45)).ToObject()}, {args: wrapArgs(ComplexType, 28.2537, -19), want: NewComplex(complex(28.2537, -19)).ToObject()}, {args: wrapArgs(ComplexType, -3.14, -0.685), want: NewComplex(complex(-3.14, -0.685)).ToObject()}, {args: wrapArgs(ComplexType, -47.234e+2, 2.374e+3), want: NewComplex(complex(-4723.4, 2374)).ToObject()}, {args: wrapArgs(ComplexType, -4.5, new(big.Int).Neg(biggestFloat)), want: NewComplex(complex(-4.5, -math.MaxFloat64)).ToObject()}, {args: wrapArgs(ComplexType, biggestFloat, biggestFloat), want: NewComplex(complex(math.MaxFloat64, math.MaxFloat64)).ToObject()}, {args: wrapArgs(ComplexType, 5, math.NaN()), want: NewComplex(complex(5, math.NaN())).ToObject()}, {args: wrapArgs(ComplexType, math.Inf(-1), -95), want: NewComplex(complex(math.Inf(-1), -95)).ToObject()}, {args: wrapArgs(ComplexType, math.NaN(), math.NaN()), want: NewComplex(complex(math.NaN(), math.NaN())).ToObject()}, {args: wrapArgs(ComplexType, math.Inf(1), math.Inf(-1)), want: NewComplex(complex(math.Inf(1), math.Inf(-1))).ToObject()}, {args: wrapArgs(ComplexType, complex(-48.8, 0.7395), 5.448), want: NewComplex(complex(-48.8, 6.1875)).ToObject()}, {args: wrapArgs(ComplexType, -3.14, complex(-4.5, -0.618)), want: NewComplex(complex(-2.5220000000000002, -4.5)).ToObject()}, {args: wrapArgs(ComplexType, complex(1, 2), complex(3, 4)), want: NewComplex(complex(-3, 5)).ToObject()}, {args: wrapArgs(ComplexType, complex(-2.47, 0.205e+2), complex(3.1, -0.4)), want: NewComplex(complex(-2.0700000000000003, 23.6)).ToObject()}, {args: wrapArgs(ComplexType, "bar", 1.2), wantExc: mustCreateException(TypeErrorType, "complex() can't take second arg if first is a string")}, {args: wrapArgs(ComplexType, "bar", None), wantExc: mustCreateException(TypeErrorType, "complex() can't take second arg if first is a string")}, {args: wrapArgs(ComplexType, 1.2, "baz"), wantExc: mustCreateException(TypeErrorType, "complex() second arg can't be a string")}, {args: wrapArgs(ComplexType, None, "baz"), wantExc: mustCreateException(TypeErrorType, "complex() second arg can't be a string")}, {args: wrapArgs(ComplexType, newObject(goodSlotType)), want: NewComplex(complex(1, 2)).ToObject()}, {args: wrapArgs(ComplexType, newObject(badSlotType)), wantExc: mustCreateException(TypeErrorType, "__complex__ returned non-complex (type object)")}, {args: wrapArgs(ComplexType, newObject(slotSubTypeType)), want: subTypeObject}, {args: wrapArgs(strictEqType, 3.14), want: newStrictEq(3.14)}, {args: wrapArgs(strictEqType, newObject(goodSlotType)), want: newStrictEq(complex(1, 2))}, {args: wrapArgs(strictEqType, newObject(badSlotType)), wantExc: mustCreateException(TypeErrorType, "__complex__ returned non-complex (type object)")}, {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'__new__' requires 1 arguments")}, {args: wrapArgs(FloatType), wantExc: mustCreateException(TypeErrorType, "complex.__new__(float): float is not a subtype of complex")}, {args: wrapArgs(ComplexType, None), wantExc: mustCreateException(TypeErrorType, "complex() argument must be a string or a number")}, {args: wrapArgs(ComplexType, "foo"), wantExc: mustCreateException(ValueErrorType, "complex() arg is a malformed string")}, {args: wrapArgs(ComplexType, 123, None, None), wantExc: mustCreateException(TypeErrorType, "'__new__' of 'complex' requires at most 2 arguments")}, } for _, cas := range cases { switch got, match := checkInvokeResult(complexNew, cas.args, cas.want, cas.wantExc); match { case checkInvokeResultExceptionMismatch: t.Errorf("complex.__new__%v raised %v, want %v", cas.args, got, cas.wantExc) case checkInvokeResultReturnValueMismatch: // Handle NaN specially, since NaN != NaN. if got == nil || cas.want == nil || !got.isInstance(ComplexType) || !cas.want.isInstance(ComplexType) || !cmplx.IsNaN(toComplexUnsafe(got).Value()) || !cmplx.IsNaN(toComplexUnsafe(cas.want).Value()) { t.Errorf("complex.__new__%v = %v, want %v", cas.args, got, cas.want) } } } } func TestComplexNonZero(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(complex(0, 0)), want: False.ToObject()}, {args: wrapArgs(complex(.0, .0)), want: False.ToObject()}, {args: wrapArgs(complex(0.0, 0.1)), want: True.ToObject()}, {args: wrapArgs(complex(1, 0)), want: True.ToObject()}, {args: wrapArgs(complex(3.14, -0.001e+5)), want: True.ToObject()}, {args: wrapArgs(complex(math.NaN(), math.NaN())), want: True.ToObject()}, {args: wrapArgs(complex(math.Inf(-1), math.Inf(1))), want: True.ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(complexNonZero), &cas); err != "" { t.Error(err) } } } func TestComplexPos(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(complex(0, 0)), want: NewComplex(complex(0, 0)).ToObject()}, {args: wrapArgs(complex(42, -0.1)), want: NewComplex(complex(42, -0.1)).ToObject()}, {args: wrapArgs(complex(-1.2, 375E+2)), want: NewComplex(complex(-1.2, 37500)).ToObject()}, {args: wrapArgs(complex(5, math.NaN())), want: NewComplex(complex(5, math.NaN())).ToObject()}, {args: wrapArgs(complex(math.Inf(1), 0.618)), want: NewComplex(complex(math.Inf(1), 0.618)).ToObject()}, } for _, cas := range cases { switch got, match := checkInvokeResult(wrapFuncForTest(complexPos), cas.args, cas.want, cas.wantExc); match { case checkInvokeResultReturnValueMismatch: if got == nil || cas.want == nil || !got.isInstance(ComplexType) || !cas.want.isInstance(ComplexType) || !complexesAreSame(toComplexUnsafe(got).Value(), toComplexUnsafe(cas.want).Value()) { t.Errorf("complex.__pos__%v = %v, want %v", cas.args, got, cas.want) } } } } func TestComplexRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(complex(0.0, 0.0)), want: NewStr("0j").ToObject()}, {args: wrapArgs(complex(0.0, 1.0)), want: NewStr("1j").ToObject()}, {args: wrapArgs(complex(1.0, 2.0)), want: NewStr("(1+2j)").ToObject()}, {args: wrapArgs(complex(3.1, -4.2)), want: NewStr("(3.1-4.2j)").ToObject()}, {args: wrapArgs(complex(math.NaN(), math.NaN())), want: NewStr("(nan+nanj)").ToObject()}, {args: wrapArgs(complex(math.Inf(-1), math.Inf(1))), want: NewStr("(-inf+infj)").ToObject()}, {args: wrapArgs(complex(math.Inf(1), math.Inf(-1))), want: NewStr("(inf-infj)").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func TestParseComplex(t *testing.T) { var ErrSyntax = errors.New("invalid syntax") cases := []struct { s string want complex128 err error }{ {"5", complex(5, 0), nil}, {"-3.14", complex(-3.14, 0), nil}, {"1.8456e3", complex(1845.6, 0), nil}, {"23j", complex(0, 23), nil}, {"7j", complex(0, 7), nil}, {"-365.12j", complex(0, -365.12), nil}, {"1+2j", complex(1, 2), nil}, {"-.3+.7j", complex(-0.3, 0.7), nil}, {"-1.3+2.7j", complex(-1.3, 2.7), nil}, {"48.39-20.3j", complex(48.39, -20.3), nil}, {"-1.23e2-30.303j", complex(-123, -30.303), nil}, {"-1.23e2-45.678e1j", complex(-123, -456.78), nil}, {"nan+nanj", complex(math.NaN(), math.NaN()), nil}, {"nan-nanj", complex(math.NaN(), math.NaN()), nil}, {"-nan-nanj", complex(math.NaN(), math.NaN()), nil}, {"inf+infj", complex(math.Inf(1), math.Inf(1)), nil}, {"inf-infj", complex(math.Inf(1), math.Inf(-1)), nil}, {"-inf-infj", complex(math.Inf(-1), math.Inf(-1)), nil}, {"infINIty+infinityj", complex(math.Inf(1), math.Inf(1)), nil}, {"3.4+j", complex(3.4, 1), nil}, {"21.98-j", complex(21.98, -1), nil}, {"+j", complex(0, 1), nil}, {"-j", complex(0, -1), nil}, {"j", complex(0, 1), nil}, {"(2.1-3.4j)", complex(2.1, -3.4), nil}, {" (2.1-3.4j) ", complex(2.1, -3.4), nil}, {" ( 2.1-3.4j ) ", complex(2.1, -3.4), nil}, {" \t \n \r ( \t \n \r 2.1-3.4j \t \n \r ) \t \n \r ", complex(2.1, -3.4), nil}, {" 3.14-15.16j ", complex(3.14, -15.16), nil}, {"(2.1-3.4j", complex(0, 0), ErrSyntax}, {"((2.1-3.4j))", complex(0, 0), ErrSyntax}, {"3.14 -15.16j", complex(0, 0), ErrSyntax}, {"3.14- 15.16j", complex(0, 0), ErrSyntax}, {"3.14-15.16 j", complex(0, 0), ErrSyntax}, {"3.14 - 15.16 j", complex(0, 0), ErrSyntax}, {"foo", complex(0, 0), ErrSyntax}, {"foo+bar", complex(0, 0), ErrSyntax}, } for _, cas := range cases { if got, _ := parseComplex(cas.s); !complexesAreSame(got, cas.want) { t.Errorf("parseComplex(%q) = %g, want %g", cas.s, got, cas.want) } } } func TestComplexHash(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(complex(0.0, 0.0)), want: NewInt(0).ToObject()}, {args: wrapArgs(complex(0.0, 1.0)), want: NewInt(1000003).ToObject()}, {args: wrapArgs(complex(1.0, 0.0)), want: NewInt(1).ToObject()}, {args: wrapArgs(complex(3.1, -4.2)), want: NewInt(-1556830019620134).ToObject()}, {args: wrapArgs(complex(3.1, 4.2)), want: NewInt(1557030815934348).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(complexHash), &cas); err != "" { t.Error(err) } } } func floatsAreSame(a, b float64) bool { return a == b || (math.IsNaN(a) && math.IsNaN(b)) } func complexesAreSame(a, b complex128) bool { return floatsAreSame(real(a), real(b)) && floatsAreSame(imag(a), imag(b)) } func tupleComplexesAreSame(got, want *Object) bool { if toTupleUnsafe(got).Len() != toTupleUnsafe(want).Len() { return false } for i := 0; i < toTupleUnsafe(got).Len(); i++ { if !complexesAreSame(toComplexUnsafe(toTupleUnsafe(got).GetItem(i)).Value(), toComplexUnsafe(toTupleUnsafe(want).GetItem(i)).Value()) { return false } } return true } ================================================ FILE: runtime/core.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "log" "reflect" "sync/atomic" ) var ( logFatal = func(msg string) { log.Fatal(msg) } // ThreadCount is the number of goroutines started with StartThread that // have not yet joined. ThreadCount int64 ) // Abs returns the result of o.__abs__ and is equivalent to the Python // expression "abs(o)". func Abs(f *Frame, o *Object) (*Object, *BaseException) { abs := o.typ.slots.Abs if abs == nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("bad operand type for abs(): '%s'", o.typ.Name())) } return abs.Fn(f, o) } // Add returns the result of adding v and w together according to the // __add/radd__ operator. func Add(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.Add, v.typ.slots.RAdd, w.typ.slots.RAdd, "+") } // And returns the result of the bitwise and operator v & w according to // __and/rand__. func And(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.And, v.typ.slots.RAnd, w.typ.slots.RAnd, "&") } // Assert raises an AssertionError if the given cond does not evaluate to true. // If msg is not nil, it is converted to a string via ToStr() and passed as args // to the raised exception. func Assert(f *Frame, cond *Object, msg *Object) *BaseException { result, raised := IsTrue(f, cond) if raised == nil && !result { if msg == nil { raised = f.Raise(AssertionErrorType.ToObject(), nil, nil) } else { var s *Str s, raised = ToStr(f, msg) if raised == nil { raised = f.RaiseType(AssertionErrorType, s.Value()) } } } return raised } // Compare implements a 3-way comparison which returns: // // -1 if v < w // 0 if v == w // 1 if v > w // // It closely resembles the behavior of CPython's do_cmp in object.c. func Compare(f *Frame, v, w *Object) (*Object, *BaseException) { cmp := v.typ.slots.Cmp if v.typ == w.typ && cmp != nil { return cmp.Fn(f, v, w) } r, raised := tryRichTo3wayCompare(f, v, w) if r != NotImplemented { return r, raised } r, raised = try3wayCompare(f, v, w) if r != NotImplemented { return r, raised } return NewInt(compareDefault(f, v, w)).ToObject(), nil } // Contains checks whether value is present in seq. It first checks the // __contains__ method of seq and, if that is not available, attempts to find // value by iteration over seq. It is equivalent to the Python expression // "value in seq". func Contains(f *Frame, seq, value *Object) (bool, *BaseException) { if contains := seq.typ.slots.Contains; contains != nil { ret, raised := contains.Fn(f, seq, value) if raised != nil { return false, raised } return IsTrue(f, ret) } iter, raised := Iter(f, seq) if raised != nil { return false, raised } o, raised := Next(f, iter) for ; raised == nil; o, raised = Next(f, iter) { eq, raised := Eq(f, o, value) if raised != nil { return false, raised } if ret, raised := IsTrue(f, eq); raised != nil { return false, raised } else if ret { return true, nil } } if !raised.isInstance(StopIterationType) { return false, raised } f.RestoreExc(nil, nil) return false, nil } // DelAttr removes the attribute of o given by name. Equivalent to the Python // expression delattr(o, name). func DelAttr(f *Frame, o *Object, name *Str) *BaseException { delAttr := o.typ.slots.DelAttr if delAttr == nil { return f.RaiseType(SystemErrorType, fmt.Sprintf("'%s' object has no __delattr__ method", o.typ.Name())) } return delAttr.Fn(f, o, name) } // DelVar removes the named variable from the given namespace dictionary such // as a module globals dict. func DelVar(f *Frame, namespace *Dict, name *Str) *BaseException { deleted, raised := namespace.DelItem(f, name.ToObject()) if raised != nil { return raised } if !deleted { return f.RaiseType(NameErrorType, fmt.Sprintf("name '%s' is not defined", name.Value())) } return nil } // DelItem performs the operation del o[key]. func DelItem(f *Frame, o, key *Object) *BaseException { delItem := o.typ.slots.DelItem if delItem == nil { return f.RaiseType(TypeErrorType, fmt.Sprintf("'%s' object does not support item deletion", o.typ.Name())) } return delItem.Fn(f, o, key) } // Div returns the result of dividing v by w according to the __div/rdiv__ // operator. func Div(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.Div, v.typ.slots.RDiv, w.typ.slots.RDiv, "/") } // DivMod returns the result (quotient and remainder tuple) of dividing v by w // according to the __divmod/rdivmod__ operator. func DivMod(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.DivMod, v.typ.slots.RDivMod, w.typ.slots.RDivMod, "divmod()") } // Eq returns the equality of v and w according to the __eq__ operator. func Eq(f *Frame, v, w *Object) (*Object, *BaseException) { r, raised := compareRich(f, compareOpEq, v, w) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } return GetBool(compareDefault(f, v, w) == 0).ToObject(), nil } // FloorDiv returns the equality of v and w according to the __floordiv/rfloordiv__ operator. func FloorDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.FloorDiv, v.typ.slots.RFloorDiv, w.typ.slots.RFloorDiv, "//") } // FormatExc calls traceback.format_exc, falling back to the single line // exception message if that fails, e.g. "NameError: name 'x' is not defined\n". func FormatExc(f *Frame) (s string) { exc, tb := f.ExcInfo() defer func() { if s == "" { strResult, raised := ToStr(f, exc.ToObject()) if raised == nil && strResult.Value() != "" { s = fmt.Sprintf("%s: %s\n", exc.typ.Name(), strResult.Value()) } else { s = exc.typ.Name() + "\n" } } f.RestoreExc(exc, tb) }() tbMod, raised := SysModules.GetItemString(f, "traceback") if raised != nil || tbMod == nil { return } formatExc, raised := GetAttr(f, tbMod, NewStr("format_exc"), nil) if raised != nil { return } result, raised := formatExc.Call(f, nil, nil) if raised != nil || !result.isInstance(StrType) { return } return toStrUnsafe(result).Value() } // GE returns the result of operation v >= w. func GE(f *Frame, v, w *Object) (*Object, *BaseException) { r, raised := compareRich(f, compareOpGE, v, w) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } return GetBool(compareDefault(f, v, w) >= 0).ToObject(), nil } // GetItem returns the result of operation o[key]. func GetItem(f *Frame, o, key *Object) (*Object, *BaseException) { getItem := o.typ.slots.GetItem if getItem == nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("'%s' object has no attribute '__getitem__'", o.typ.Name())) } return getItem.Fn(f, o, key) } // GetAttr returns the named attribute of o. Equivalent to the Python expression // getattr(o, name, def). func GetAttr(f *Frame, o *Object, name *Str, def *Object) (*Object, *BaseException) { // TODO: Fall back to __getattr__. getAttribute := o.typ.slots.GetAttribute if getAttribute == nil { msg := fmt.Sprintf("'%s' has no attribute '%s'", o.typ.Name(), name.Value()) return nil, f.RaiseType(AttributeErrorType, msg) } result, raised := getAttribute.Fn(f, o, name) if raised != nil && raised.isInstance(AttributeErrorType) && def != nil { f.RestoreExc(nil, nil) result, raised = def, nil } return result, raised } // GT returns the result of operation v > w. func GT(f *Frame, v, w *Object) (*Object, *BaseException) { r, raised := compareRich(f, compareOpGT, v, w) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } return GetBool(compareDefault(f, v, w) > 0).ToObject(), nil } // Hash returns the hash of o according to its __hash__ operator. func Hash(f *Frame, o *Object) (*Int, *BaseException) { hash := o.typ.slots.Hash if hash == nil { _, raised := hashNotImplemented(f, o) return nil, raised } h, raised := hash.Fn(f, o) if raised != nil { return nil, raised } if !h.isInstance(IntType) { return nil, f.RaiseType(TypeErrorType, "an integer is required") } return toIntUnsafe(h), nil } // Hex returns the result of o.__hex__ if defined. func Hex(f *Frame, o *Object) (*Object, *BaseException) { hex := o.typ.slots.Hex if hex == nil { raised := f.RaiseType(TypeErrorType, "hex() argument can't be converted to hex") return nil, raised } h, raised := hex.Fn(f, o) if raised != nil { return nil, raised } if !h.isInstance(StrType) { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("__hex__ returned non-string (type %s)", h.typ.name)) } return h, nil } // IAdd returns the result of v.__iadd__ if defined, otherwise falls back to // Add. func IAdd(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IAdd, Add) } // IAnd returns the result of v.__iand__ if defined, otherwise falls back to // And. func IAnd(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IAnd, And) } // IDiv returns the result of v.__idiv__ if defined, otherwise falls back to // div. func IDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IDiv, Div) } // IFloorDiv returns the result of v.__ifloordiv__ if defined, otherwise falls back to // floordiv. func IFloorDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IFloorDiv, FloorDiv) } // ILShift returns the result of v.__ilshift__ if defined, otherwise falls back // to lshift. func ILShift(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.ILShift, LShift) } // IMod returns the result of v.__imod__ if defined, otherwise falls back to // mod. func IMod(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IMod, Mod) } // IMul returns the result of v.__imul__ if defined, otherwise falls back to // mul. func IMul(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IMul, Mul) } // Invert returns the result of o.__invert__ and is equivalent to the Python // expression "~o". func Invert(f *Frame, o *Object) (*Object, *BaseException) { invert := o.typ.slots.Invert if invert == nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("bad operand type for unary ~: '%s'", o.typ.Name())) } return invert.Fn(f, o) } // IOr returns the result of v.__ior__ if defined, otherwise falls back to Or. func IOr(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IOr, Or) } // IPow returns the result of v.__pow__ if defined, otherwise falls back to IPow. func IPow(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IPow, Pow) } // IRShift returns the result of v.__irshift__ if defined, otherwise falls back // to rshift. func IRShift(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IRShift, RShift) } // IsInstance returns true if the type o is an instance of classinfo, or an // instance of an element in classinfo (if classinfo is a tuple). It returns // false otherwise. The argument classinfo must be a type or a tuple whose // elements are types like the isinstance() Python builtin. func IsInstance(f *Frame, o *Object, classinfo *Object) (bool, *BaseException) { return IsSubclass(f, o.typ.ToObject(), classinfo) } // IsSubclass returns true if the type o is a subtype of classinfo or a subtype // of an element in classinfo (if classinfo is a tuple). It returns false // otherwise. The argument o must be a type and classinfo must be a type or a // tuple whose elements are types like the issubclass() Python builtin. func IsSubclass(f *Frame, o *Object, classinfo *Object) (bool, *BaseException) { if !o.isInstance(TypeType) { return false, f.RaiseType(TypeErrorType, "issubclass() arg 1 must be a class") } t := toTypeUnsafe(o) errorMsg := "classinfo must be a type or tuple of types" if classinfo.isInstance(TypeType) { return t.isSubclass(toTypeUnsafe(classinfo)), nil } if !classinfo.isInstance(TupleType) { return false, f.RaiseType(TypeErrorType, errorMsg) } for _, elem := range toTupleUnsafe(classinfo).elems { if !elem.isInstance(TypeType) { return false, f.RaiseType(TypeErrorType, errorMsg) } if t.isSubclass(toTypeUnsafe(elem)) { return true, nil } } return false, nil } // IsTrue returns the truthiness of o according to the __nonzero__ operator. func IsTrue(f *Frame, o *Object) (bool, *BaseException) { switch o { case True.ToObject(): return true, nil case False.ToObject(), None: return false, nil } nonzero := o.typ.slots.NonZero if nonzero != nil { r, raised := nonzero.Fn(f, o) if raised != nil { return false, raised } if r.isInstance(IntType) { return toIntUnsafe(r).IsTrue(), nil } msg := fmt.Sprintf("__nonzero__ should return bool, returned %s", r.typ.Name()) return false, f.RaiseType(TypeErrorType, msg) } if o.typ.slots.Len != nil { l, raised := Len(f, o) if raised != nil { return false, raised } return l.IsTrue(), nil } return true, nil } // ISub returns the result of v.__isub__ if defined, otherwise falls back to // sub. func ISub(f *Frame, v, w *Object) (*Object, *BaseException) { if isub := v.typ.slots.ISub; isub != nil { return isub.Fn(f, v, w) } return Sub(f, v, w) } // Iter implements the Python iter() builtin. It returns an iterator for o if // o is iterable. Otherwise it raises TypeError. // Note that the iter(f, sentinel) form is not yet supported. func Iter(f *Frame, o *Object) (*Object, *BaseException) { // TODO: Support iter(f, sentinel) usage. iter := o.typ.slots.Iter if iter != nil { return iter.Fn(f, o) } if o.typ.slots.GetItem != nil { return newSeqIterator(o), nil } return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("'%s' object is not iterable", o.typ.Name())) } // IXor returns the result of v.__ixor__ if defined, otherwise falls back to // Xor. func IXor(f *Frame, v, w *Object) (*Object, *BaseException) { return inplaceOp(f, v, w, v.typ.slots.IXor, Xor) } // LE returns the result of operation v <= w. func LE(f *Frame, v, w *Object) (*Object, *BaseException) { r, raised := compareRich(f, compareOpLE, v, w) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } return GetBool(compareDefault(f, v, w) <= 0).ToObject(), nil } // Len returns the length of the given sequence object. func Len(f *Frame, o *Object) (*Int, *BaseException) { lenSlot := o.typ.slots.Len if lenSlot == nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("object of type '%s' has no len()", o.typ.Name())) } r, raised := lenSlot.Fn(f, o) if raised != nil { return nil, raised } if !r.isInstance(IntType) { return nil, f.RaiseType(TypeErrorType, "an integer is required") } return toIntUnsafe(r), nil } // LShift returns the result of v << w according to the __lshift/rlshift__ // operator. func LShift(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.LShift, v.typ.slots.RLShift, w.typ.slots.RLShift, "<<") } // LT returns the result of operation v < w. func LT(f *Frame, v, w *Object) (*Object, *BaseException) { r, raised := compareRich(f, compareOpLT, v, w) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } return GetBool(compareDefault(f, v, w) < 0).ToObject(), nil } // Mod returns the remainder from the division of v by w according to the // __mod/rmod__ operator. func Mod(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.Mod, v.typ.slots.RMod, w.typ.slots.RMod, "%") } // Mul returns the result of multiplying v and w together according to the // __mul/rmul__ operator. func Mul(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.Mul, v.typ.slots.RMul, w.typ.slots.RMul, "*") } // Pow returns the result of x**y, the base-x exponential of y according to the // __pow/rpow__ operator. func Pow(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.Pow, v.typ.slots.RPow, w.typ.slots.RPow, "**") } // Or returns the result of the bitwise or operator v | w according to // __or/ror__. func Or(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.Or, v.typ.slots.ROr, w.typ.slots.ROr, "|") } // Index returns the o converted to a Python int or long according to o's // __index__ slot. func Index(f *Frame, o *Object) (*Object, *BaseException) { if o.isInstance(IntType) || o.isInstance(LongType) { return o, nil } index := o.typ.slots.Index if index == nil { return nil, nil } i, raised := index.Fn(f, o) if raised != nil { return nil, raised } if !i.isInstance(IntType) && !i.isInstance(LongType) { format := "__index__ returned non-(int,long) (type %s)" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, i.typ.Name())) } return i, nil } // IndexInt returns the value of o converted to a Go int according to o's // __index__ slot. // It raises a TypeError if o doesn't have an __index__ method. func IndexInt(f *Frame, o *Object) (i int, raised *BaseException) { if index := o.typ.slots.Index; index != nil { // Unwrap __index__ slot and fall through. o, raised = index.Fn(f, o) if raised != nil { return 0, raised } } if o.isInstance(IntType) { return toIntUnsafe(o).Value(), nil } if o.isInstance(LongType) { l := toLongUnsafe(o).Value() // Anything bigger than maxIntBig will treat as maxIntBig. if !numInIntRange(l) { l = maxIntBig } return int(l.Int64()), nil } return 0, f.RaiseType(TypeErrorType, errBadSliceIndex) } // Invoke calls the given callable with the positional arguments given by args // and *varargs, and the keyword arguments by keywords and **kwargs. It first // packs the arguments into slices for the positional and keyword arguments, // then it passes those to *Object.Call. func Invoke(f *Frame, callable *Object, args Args, varargs *Object, keywords KWArgs, kwargs *Object) (*Object, *BaseException) { if varargs != nil { raised := seqApply(f, varargs, func(elems []*Object, _ bool) *BaseException { numArgs := len(args) packed := make([]*Object, numArgs+len(elems)) copy(packed, args) copy(packed[numArgs:], elems) args = packed return nil }) if raised != nil { return nil, raised } } if kwargs != nil { if !kwargs.isInstance(DictType) { format := "argument after ** must be a dict, not %s" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, kwargs.typ.Name())) } kwargsDict := toDictUnsafe(kwargs) numKeywords := len(keywords) numKwargs, raised := Len(f, kwargs) if raised != nil { return nil, raised } // Don't bother synchronizing access to len(kwargs) since it's just a // hint and it doesn't matter if it's a little off. packed := make(KWArgs, numKeywords, numKeywords+numKwargs.Value()) copy(packed, keywords) raised = seqForEach(f, kwargs, func(o *Object) *BaseException { if !o.isInstance(StrType) { return f.RaiseType(TypeErrorType, "keywords must be strings") } s := toStrUnsafe(o).Value() // Search for dupes linearly assuming small number of keywords. for _, kw := range keywords { if kw.Name == s { format := "got multiple values for keyword argument '%s'" return f.RaiseType(TypeErrorType, fmt.Sprintf(format, s)) } } item, raised := kwargsDict.GetItem(f, o) if raised != nil { return raised } if item == nil { return raiseKeyError(f, o) } packed = append(packed, KWArg{Name: s, Value: item}) return nil }) if raised != nil { return nil, raised } keywords = packed } return callable.Call(f, args, keywords) } // NE returns the non-equality of v and w according to the __ne__ operator. func NE(f *Frame, v, w *Object) (*Object, *BaseException) { r, raised := compareRich(f, compareOpNE, v, w) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } return GetBool(compareDefault(f, v, w) != 0).ToObject(), nil } // Next implements the Python next() builtin. It calls next on the provided // iterator. It raises TypeError if iter is not an iterator object. // Note that the next(it, default) form is not yet supported. func Next(f *Frame, iter *Object) (*Object, *BaseException) { // TODO: Support next(it, default) usage. next := iter.typ.slots.Next if next == nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("%s object is not an iterator", iter.typ.Name())) } return next.Fn(f, iter) } // Oct returns the result of o.__oct__ if defined. func Oct(f *Frame, o *Object) (*Object, *BaseException) { oct := o.typ.slots.Oct if oct == nil { raised := f.RaiseType(TypeErrorType, "oct() argument can't be converted to oct") return nil, raised } o, raised := oct.Fn(f, o) if raised != nil { return nil, raised } if !o.isInstance(StrType) { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("__oct__ returned non-string (type %s)", o.typ.name)) } return o, nil } // Pos returns the result of o.__pos__ and is equivalent to the Python // expression "+o". func Pos(f *Frame, o *Object) (*Object, *BaseException) { pos := o.typ.slots.Pos if pos == nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("bad operand type for unary +: '%s'", o.typ.Name())) } return pos.Fn(f, o) } // Print implements the Python print statement. It calls str() on the given args // and outputs the results to stdout separated by spaces. Similar to the Python // print statement. func Print(f *Frame, args Args, nl bool) *BaseException { // TODO: Support outputting to files other than stdout and softspace. var end string if nl { end = "\n" } else if len(args) > 0 { end = " " } return pyPrint(f, args, " ", end, Stdout) } // Repr returns a string containing a printable representation of o. This is // equivalent to the Python expression "repr(o)". func Repr(f *Frame, o *Object) (*Str, *BaseException) { repr := o.typ.slots.Repr if repr == nil { s, raised := o.typ.FullName(f) if raised != nil { return nil, raised } return NewStr(fmt.Sprintf("<%s object at %p>", s, o)), nil } r, raised := repr.Fn(f, o) if raised != nil { return nil, raised } if !r.isInstance(StrType) { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("__repr__ returned non-string (type %s)", r.typ.Name())) } return toStrUnsafe(r), nil } // ResolveClass resolves name in the class dict given by class, falling back to // the provided local if it is non-nil, otherwise falling back to globals. // This is used by the code generator to resolve names in the context of a class // definition. If the class definition occurs in a closure in which a local of // the given name is present then local will be non-nil, otherwise it will be // nil. func ResolveClass(f *Frame, class *Dict, local *Object, name *Str) (*Object, *BaseException) { if value, raised := class.GetItem(f, name.ToObject()); raised != nil || value != nil { return value, raised } if local != nil { if raised := CheckLocal(f, local, name.Value()); raised != nil { return nil, raised } return local, nil } return ResolveGlobal(f, name) } // ResolveGlobal looks up name in the frame's dict of global variables or in // the Builtins dict if absent. It raises NameError when absent from both. func ResolveGlobal(f *Frame, name *Str) (*Object, *BaseException) { if value, raised := f.Globals().GetItem(f, name.ToObject()); raised != nil || value != nil { return value, raised } value, raised := Builtins.GetItem(f, name.ToObject()) if raised != nil { return nil, raised } if value == nil { return nil, f.RaiseType(NameErrorType, fmt.Sprintf("name '%s' is not defined", name.Value())) } return value, nil } // RShift returns the result of v >> w according to the __rshift/rrshift__ // operator. func RShift(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.RShift, v.typ.slots.RRShift, w.typ.slots.RRShift, ">>") } // CheckLocal validates that the local variable with the given name and value // has been bound and raises UnboundLocalError if not. func CheckLocal(f *Frame, value *Object, name string) *BaseException { if value == UnboundLocal { format := "local variable '%s' referenced before assignment" return f.RaiseType(UnboundLocalErrorType, fmt.Sprintf(format, name)) } return nil } // SetAttr sets the attribute of o given by name to value. Equivalent to the // Python expression setattr(o, name, value). func SetAttr(f *Frame, o *Object, name *Str, value *Object) *BaseException { setAttr := o.typ.slots.SetAttr if setAttr == nil { return f.RaiseType(SystemErrorType, fmt.Sprintf("'%s' object has no __setattr__ method", o.typ.Name())) } return setAttr.Fn(f, o, name, value) } // SetItem performs the operation o[key] = value. func SetItem(f *Frame, o, key, value *Object) *BaseException { setItem := o.typ.slots.SetItem if setItem == nil { return f.RaiseType(TypeErrorType, fmt.Sprintf("'%s' object has no attribute '__setitem__'", o.typ.Name())) } return setItem.Fn(f, o, key, value) } // StartThread runs callable in a new goroutine. func StartThread(callable *Object) { go func() { atomic.AddInt64(&ThreadCount, 1) defer atomic.AddInt64(&ThreadCount, -1) f := NewRootFrame() _, raised := callable.Call(f, nil, nil) if raised != nil { Stderr.writeString(FormatExc(f)) } }() } // Sub returns the result of subtracting v from w according to the // __sub/rsub__ operator. func Sub(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.Sub, v.typ.slots.RSub, w.typ.slots.RSub, "-") } // TieTarget is a data structure used to facilitate iterator unpacking in // assignment statements. A TieTarget should have one of Target or Children // populated but not both. // // As an example, the targets in the Python assignment 'foo, bar = ...' // could be represented as: // // TieTarget{ // Children: []TieTarget{{Target: &foo}, {Target: &bar}}, // } type TieTarget struct { // Target is a destination pointer where an unpacked value will be // stored. Target **Object // Children contains a sequence of TieTargets that should be unpacked // into. Children []TieTarget } // Tie takes a (possibly nested) TieTarget and recursively unpacks the // elements of o by iteration, assigning the results to the Target fields of t. // If the structure of o is not suitable to be unpacked into t, then an // exception is raised. func Tie(f *Frame, t TieTarget, o *Object) *BaseException { if t.Target != nil { *t.Target = o return nil } iter, raised := Iter(f, o) if raised != nil { return raised } for i, child := range t.Children { if value, raised := Next(f, iter); raised == nil { if raised := Tie(f, child, value); raised != nil { return raised } } else if raised.isInstance(StopIterationType) { return f.RaiseType(ValueErrorType, fmt.Sprintf("need more than %d values to unpack", i)) } else { return raised } } _, raised = Next(f, iter) if raised == nil { return f.RaiseType(ValueErrorType, "too many values to unpack") } if !raised.isInstance(StopIterationType) { return raised } f.RestoreExc(nil, nil) return nil } // ToInt converts o to an integer type according to the __int__ slot. If the // result is not an int or long, then an exception is raised. func ToInt(f *Frame, o *Object) (*Object, *BaseException) { if o.typ == IntType || o.typ == LongType { return o, nil } intSlot := o.typ.slots.Int if intSlot == nil { return nil, f.RaiseType(TypeErrorType, "an integer is required") } i, raised := intSlot.Fn(f, o) if raised != nil { return nil, raised } if i.isInstance(IntType) || i.isInstance(LongType) { return i, nil } return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("__int__ returned non-int (type %s)", i.typ.Name())) } // ToIntValue converts o to an integer according to the __int__ slot. If the // result is not an int or long, or if the long value is too large to fit into // an int, then an exception is raised. func ToIntValue(f *Frame, o *Object) (int, *BaseException) { i, raised := ToInt(f, o) if raised != nil { return 0, raised } if i.isInstance(IntType) { return toIntUnsafe(i).Value(), nil } return toLongUnsafe(i).IntValue(f) } // ToNative converts o to a native Go object according to the __native__ // operator. func ToNative(f *Frame, o *Object) (reflect.Value, *BaseException) { if native := o.typ.slots.Native; native != nil { return native.Fn(f, o) } return reflect.ValueOf(o), nil } // ToStr is a convenience function for calling "str(o)". func ToStr(f *Frame, o *Object) (*Str, *BaseException) { result, raised := StrType.Call(f, []*Object{o}, nil) if raised != nil { return nil, raised } if !result.isInstance(StrType) { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("__str__ returned non-string (type %s)", result.typ.Name())) } return toStrUnsafe(result), nil } // Neg returns the result of o.__neg__ and is equivalent to the Python // expression "-o". func Neg(f *Frame, o *Object) (*Object, *BaseException) { neg := o.typ.slots.Neg if neg == nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("bad operand type for unary -: '%s'", o.typ.Name())) } return neg.Fn(f, o) } // Xor returns the result of the bitwise xor operator v ^ w according to // __xor/rxor__. func Xor(f *Frame, v, w *Object) (*Object, *BaseException) { return binaryOp(f, v, w, v.typ.slots.Xor, v.typ.slots.RXor, w.typ.slots.RXor, "^") } const ( errResultTooLarge = "result too large" errUnsupportedOperand = "unsupported operand type(s) for %s: '%s' and '%s'" ) // binaryOp picks an appropriate operator method (op or rop) from v or w and // returns its result. It raises TypeError if no appropriate method is found. // It is similar to CPython's binary_op1 function from abstract.c. func binaryOp(f *Frame, v, w *Object, op, vrop, wrop *binaryOpSlot, opName string) (*Object, *BaseException) { if v.typ != w.typ && w.typ.isSubclass(v.typ) { // w is an instance of a subclass of type(v), so prefer w's more // specific rop, but only if it is overridden (wrop != vrop). if wrop != nil && wrop != vrop { r, raised := wrop.Fn(f, w, v) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } } } if op != nil { r, raised := op.Fn(f, v, w) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } } if wrop != nil { r, raised := wrop.Fn(f, w, v) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } } return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(errUnsupportedOperand, opName, v.typ.Name(), w.typ.Name())) } func inplaceOp(f *Frame, v, w *Object, slot *binaryOpSlot, fallback binaryOpFunc) (*Object, *BaseException) { if slot != nil { return slot.Fn(f, v, w) } return fallback(f, v, w) } type compareOp int const ( compareOpLT compareOp = iota compareOpLE compareOpEq compareOpNE compareOpGE compareOpGT ) var compareOpSwapped = []compareOp{ compareOpGT, compareOpGE, compareOpEq, compareOpNE, compareOpLE, compareOpLT, } func (op compareOp) swapped() compareOp { return compareOpSwapped[op] } func (op compareOp) slot(t *Type) *binaryOpSlot { switch op { case compareOpLT: return t.slots.LT case compareOpLE: return t.slots.LE case compareOpEq: return t.slots.Eq case compareOpNE: return t.slots.NE case compareOpGE: return t.slots.GE case compareOpGT: return t.slots.GT } panic(fmt.Sprintf("invalid compareOp value: %d", op)) } func compareRich(f *Frame, op compareOp, v, w *Object) (*Object, *BaseException) { r, raised := tryRichCompare(f, op, v, w) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } return try3wayToRichCompare(f, op, v, w) } // convert3wayToObject converts the integer results from a 3-way // comparison to a suitable boolean value for the given rich // comparison op. func convert3wayToObject(op compareOp, c int) *Object { b := false switch op { case compareOpLT: b = c < 0 case compareOpLE: b = c <= 0 case compareOpEq: b = c == 0 case compareOpNE: b = c != 0 case compareOpGE: b = c >= 0 case compareOpGT: b = c > 0 } return GetBool(b).ToObject() } // try3wayToRichCompare tries to perform a rich comparison operation on the given objects // with the given comparison op using 3-way comparison. It closely resembles the behavior // of CPython's try_3way_to_rich_compare in object.c. func try3wayToRichCompare(f *Frame, op compareOp, v, w *Object) (*Object, *BaseException) { r, raised := try3wayCompare(f, v, w) if raised != nil { return nil, raised } c := 0 if r == NotImplemented { c = compareDefault(f, v, w) } else { c = toIntUnsafe(r).Value() } return convert3wayToObject(op, c), nil } // tryRichCompare tries to perform a rich comparison operation on the given // objects with the given comparison op using the rich comparison methods. // It closely resembles the behavior of CPython's try_rich_compare in object.c. func tryRichCompare(f *Frame, op compareOp, v, w *Object) (*Object, *BaseException) { if v.typ != w.typ && w.typ.isSubclass(v.typ) { // type(w) is a subclass of type(v) so try to use w's // comparison operators since they're more specific. slot := op.swapped().slot(w.typ) if slot != nil { r, raised := slot.Fn(f, w, v) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } } } slot := op.slot(v.typ) if slot != nil { r, raised := slot.Fn(f, v, w) if raised != nil { return nil, raised } if r != NotImplemented { return r, nil } } slot = op.swapped().slot(w.typ) if slot != nil { return slot.Fn(f, w, v) } return NotImplemented, nil } // compareDefault returns is the fallback logic for object comparison. It // closely resembles the behavior of CPython's default_3way_compare in object.c. func compareDefault(f *Frame, v, w *Object) int { if v.typ == w.typ { pv, pw := uintptr(v.toPointer()), uintptr(w.toPointer()) if pv < pw { return -1 } if pv == pw { return 0 } return 1 } if v == None { return -1 } if w == None { return 1 } // TODO: In default_3way_compare, the number type name is the empty // string so it evaluates less than non-number types. Once Grumpy // supports the concept of number types, add this behavior. if v.typ.Name() < w.typ.Name() { return -1 } if v.typ.Name() != w.typ.Name() { return 1 } if uintptr(v.typ.toPointer()) < uintptr(w.typ.toPointer()) { return -1 } return 1 } // tryRichCompareBool tries a rich comparison with the given comparison op and // returns a bool indicating if the relation is true. It closely resembles the // behavior of CPython's try_rich_compare_bool in object.c. func tryRichCompareBool(f *Frame, op compareOp, v, w *Object) (bool, *BaseException) { r, raised := tryRichCompare(f, op, v, w) if raised != nil { return false, raised } if r == NotImplemented { return false, nil } br, raised := IsTrue(f, r) if raised != nil { return false, raised } return br, raised } // halfCompare tries a comparison with the __cmp__ slot, ensures the result // is an integer, and returns it. It closely resembles the behavior of CPython's // half_compare in typeobject.c. func halfCompare(f *Frame, v, w *Object) (*Object, *BaseException) { cmp := v.typ.slots.Cmp r, raised := cmp.Fn(f, v, w) if raised != nil { return nil, raised } if !r.isInstance(IntType) { return nil, f.RaiseType(TypeErrorType, "an integer is required") } return r, nil } // try3wayCompare tries a comparison with the __cmp__ slot with the given // arguments. It first tries to use the __cmp__ slot on v and if that fails // on w. It closely resembles the behavior of CPython's try_3way_compare in // object.c. func try3wayCompare(f *Frame, v, w *Object) (*Object, *BaseException) { cmp := v.typ.slots.Cmp if cmp != nil { return halfCompare(f, v, w) } cmp = w.typ.slots.Cmp if cmp != nil { r, raised := halfCompare(f, w, v) if raised != nil { return nil, raised } return intNeg(f, r) } return NotImplemented, nil } // tryRichTo3wayCompare tries to compute a 3-way comparison in terms of // the rich comparison operators (if they exist). It closely resembles // the behavior of CPython's try_rich_to_3way_compare in object.c. func tryRichTo3wayCompare(f *Frame, v, w *Object) (*Object, *BaseException) { var tries = []struct { op compareOp outcome int }{ {compareOpEq, 0}, {compareOpLT, -1}, {compareOpGT, 1}, } for _, try := range tries { r, raised := tryRichCompareBool(f, try.op, v, w) if raised != nil { return nil, raised } if r { return NewInt(try.outcome).ToObject(), nil } } return NotImplemented, nil } func checkFunctionArgs(f *Frame, function string, args Args, types ...*Type) *BaseException { if len(args) != len(types) { msg := fmt.Sprintf("'%s' requires %d arguments", function, len(types)) return f.RaiseType(TypeErrorType, msg) } for i, t := range types { if !args[i].isInstance(t) { format := "'%s' requires a '%s' object but received a %q" return f.RaiseType(TypeErrorType, fmt.Sprintf(format, function, t.Name(), args[i].typ.Name())) } } return nil } func checkFunctionVarArgs(f *Frame, function string, args Args, types ...*Type) *BaseException { if len(args) <= len(types) { return checkFunctionArgs(f, function, args, types...) } return checkFunctionArgs(f, function, args[:len(types)], types...) } func checkMethodArgs(f *Frame, method string, args Args, types ...*Type) *BaseException { if len(args) != len(types) { msg := fmt.Sprintf("'%s' of '%s' requires %d arguments", method, types[0].Name(), len(types)) return f.RaiseType(TypeErrorType, msg) } for i, t := range types { if !args[i].isInstance(t) { format := "'%s' requires a '%s' object but received a '%s'" return f.RaiseType(TypeErrorType, fmt.Sprintf(format, method, t.Name(), args[i].typ.Name())) } } return nil } func checkMethodVarArgs(f *Frame, method string, args Args, types ...*Type) *BaseException { if len(args) <= len(types) { return checkMethodArgs(f, method, args, types...) } return checkMethodArgs(f, method, args[:len(types)], types...) } func hashNotImplemented(f *Frame, o *Object) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("unhashable type: '%s'", o.typ.Name())) } // pyPrint encapsulates the logic of the Python print function. func pyPrint(f *Frame, args Args, sep, end string, file *File) *BaseException { for i, arg := range args { if i > 0 { err := file.writeString(sep) if err != nil { return f.RaiseType(IOErrorType, err.Error()) } } s, raised := ToStr(f, arg) if raised != nil { return raised } err := file.writeString(s.Value()) if err != nil { return f.RaiseType(IOErrorType, err.Error()) } } err := file.writeString(end) if err != nil { return f.RaiseType(IOErrorType, err.Error()) } return nil } ================================================ FILE: runtime/core_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "math/big" "reflect" "regexp" "runtime" "testing" ) func TestAssert(t *testing.T) { assert := newBuiltinFunction("TestAssert", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { switch argc := len(args); argc { case 1: if raised := Assert(f, args[0], nil); raised != nil { return nil, raised } case 2: if raised := Assert(f, args[0], args[1]); raised != nil { return nil, raised } default: return nil, f.RaiseType(SystemErrorType, fmt.Sprintf("Assert expected 1 or 2 args, got %d", argc)) } return None, nil }).ToObject() emptyAssert := toBaseExceptionUnsafe(mustNotRaise(AssertionErrorType.Call(NewRootFrame(), nil, nil))) cases := []invokeTestCase{ {args: wrapArgs(true), want: None}, {args: wrapArgs(NewTuple(None)), want: None}, {args: wrapArgs(None), wantExc: emptyAssert}, {args: wrapArgs(NewDict()), wantExc: emptyAssert}, {args: wrapArgs(false, "foo"), wantExc: mustCreateException(AssertionErrorType, "foo")}, } for _, cas := range cases { if err := runInvokeTestCase(assert, &cas); err != "" { t.Error(err) } } } func TestBinaryOps(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__add__": newBuiltinFunction("__add__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("foo add").ToObject(), nil }).ToObject(), "__radd__": newBuiltinFunction("__add__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("foo radd").ToObject(), nil }).ToObject(), })) barType := newTestClass("Bar", []*Type{fooType}, NewDict()) bazType := newTestClass("Baz", []*Type{IntType}, newStringDict(map[string]*Object{ "__rdiv__": newBuiltinFunction("__rdiv__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { s, raised := ToStr(f, args[1]) if raised != nil { return nil, raised } return s.ToObject(), nil }).ToObject(), })) inplaceType := newTestClass("Inplace", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__iadd__": newBuiltinFunction("__iadd__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), "__iand__": newBuiltinFunction("__iand__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), "__idiv__": newBuiltinFunction("__idiv__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), "__ilshift__": newBuiltinFunction("__ilshift__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), "__imod__": newBuiltinFunction("__imod__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), "__imul__": newBuiltinFunction("__imul__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), "__ior__": newBuiltinFunction("__ior__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), "__irshift__": newBuiltinFunction("__irshift__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), "__isub__": newBuiltinFunction("__isub__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), "__ixor__": newBuiltinFunction("__ixor__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[1], nil }).ToObject(), })) cases := []struct { fun func(f *Frame, v, w *Object) (*Object, *BaseException) v, w *Object want *Object wantExc *BaseException }{ {Add, NewStr("foo").ToObject(), NewStr("bar").ToObject(), NewStr("foobar").ToObject(), nil}, {Add, NewStr("foo").ToObject(), NewStr("bar").ToObject(), NewStr("foobar").ToObject(), nil}, {Add, newObject(fooType), newObject(ObjectType), NewStr("foo add").ToObject(), nil}, {And, NewInt(-42).ToObject(), NewInt(244).ToObject(), NewInt(212).ToObject(), nil}, {And, NewInt(42).ToObject(), NewStr("foo").ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for &: 'int' and 'str'")}, {Add, newObject(fooType), newObject(barType), NewStr("foo add").ToObject(), nil}, {Div, NewInt(123).ToObject(), newObject(bazType), NewStr("123").ToObject(), nil}, {IAdd, NewStr("foo").ToObject(), NewStr("bar").ToObject(), NewStr("foobar").ToObject(), nil}, {IAdd, NewStr("foo").ToObject(), NewStr("bar").ToObject(), NewStr("foobar").ToObject(), nil}, {IAdd, newObject(fooType), newObject(ObjectType), NewStr("foo add").ToObject(), nil}, {IAdd, newObject(inplaceType), NewStr("foo").ToObject(), NewStr("foo").ToObject(), nil}, {IAnd, NewInt(9).ToObject(), NewInt(12).ToObject(), NewInt(8).ToObject(), nil}, {IAnd, newObject(inplaceType), NewStr("foo").ToObject(), NewStr("foo").ToObject(), nil}, {IAnd, newObject(ObjectType), newObject(fooType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for &: 'object' and 'Foo'")}, {IDiv, NewInt(123).ToObject(), newObject(bazType), NewStr("123").ToObject(), nil}, {IDiv, newObject(inplaceType), NewInt(42).ToObject(), NewInt(42).ToObject(), nil}, {ILShift, newObject(inplaceType), NewInt(123).ToObject(), NewInt(123).ToObject(), nil}, {IMod, NewInt(24).ToObject(), NewInt(6).ToObject(), NewInt(0).ToObject(), nil}, {IMod, newObject(inplaceType), NewFloat(3.14).ToObject(), NewFloat(3.14).ToObject(), nil}, {IMul, NewStr("foo").ToObject(), NewInt(3).ToObject(), NewStr("foofoofoo").ToObject(), nil}, {IMul, newObject(inplaceType), True.ToObject(), True.ToObject(), nil}, {IMul, newObject(ObjectType), newObject(fooType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'object' and 'Foo'")}, {IOr, newObject(inplaceType), NewInt(42).ToObject(), NewInt(42).ToObject(), nil}, {IOr, NewInt(9).ToObject(), NewInt(12).ToObject(), NewInt(13).ToObject(), nil}, {IOr, newObject(ObjectType), newObject(fooType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for |: 'object' and 'Foo'")}, {IRShift, newObject(inplaceType), NewInt(123).ToObject(), NewInt(123).ToObject(), nil}, {ISub, NewInt(3).ToObject(), NewInt(-3).ToObject(), NewInt(6).ToObject(), nil}, {ISub, newObject(inplaceType), None, None, nil}, {IXor, newObject(inplaceType), None, None, nil}, {IXor, NewInt(9).ToObject(), NewInt(12).ToObject(), NewInt(5).ToObject(), nil}, {IXor, newObject(ObjectType), newObject(fooType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for ^: 'object' and 'Foo'")}, {Mod, NewInt(24).ToObject(), NewInt(6).ToObject(), NewInt(0).ToObject(), nil}, {Mul, NewStr("foo").ToObject(), NewInt(3).ToObject(), NewStr("foofoofoo").ToObject(), nil}, {Mul, newObject(ObjectType), newObject(fooType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'object' and 'Foo'")}, {Or, NewInt(-42).ToObject(), NewInt(244).ToObject(), NewInt(-10).ToObject(), nil}, {Or, NewInt(42).ToObject(), NewStr("foo").ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for |: 'int' and 'str'")}, {Pow, NewInt(2).ToObject(), NewInt(-2).ToObject(), NewFloat(0.25).ToObject(), nil}, {Pow, NewInt(2).ToObject(), newObject(fooType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for **: 'int' and 'Foo'")}, {Sub, NewInt(3).ToObject(), NewInt(-3).ToObject(), NewInt(6).ToObject(), nil}, {Xor, NewInt(-42).ToObject(), NewInt(244).ToObject(), NewInt(-222).ToObject(), nil}, {Xor, NewInt(42).ToObject(), NewStr("foo").ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for ^: 'int' and 'str'")}, } for _, cas := range cases { testCase := invokeTestCase{wrapArgs(cas.v, cas.w), nil, cas.want, cas.wantExc} if err := runInvokeTestCase(wrapFuncForTest(cas.fun), &testCase); err != "" { t.Error(err) } } } func TestCompare(t *testing.T) { badCmpType := newTestClass("BadCmp", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__cmp__": newBuiltinFunction("__cmp__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, "uh oh") }).ToObject(), })) cmpLtType := newTestClass("Lt", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__cmp__": newBuiltinFunction("__cmp__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewInt(-1).ToObject(), nil }).ToObject(), })) cmpEqType := newTestClass("Eq", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__cmp__": newBuiltinFunction("__cmp__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewInt(0).ToObject(), nil }).ToObject(), })) cmpGtType := newTestClass("Gt", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__cmp__": newBuiltinFunction("__cmp__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewInt(1).ToObject(), nil }).ToObject(), })) cmpByEqType := newTestClass("EqCmp", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return True.ToObject(), nil }).ToObject(), })) badCmpByEqType := newTestClass("BadEqCmp", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, "uh oh") }).ToObject(), })) badNonZeroType := newTestClass("BadNonZeroType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__nonzero__": newBuiltinFunction("__nonzero__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, "uh oh") }).ToObject(), })) worseCmpByEqType := newTestClass("WorseEqCmp", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return newObject(badNonZeroType), nil }).ToObject(), })) cmpNonIntResultType := newTestClass("CmpNonIntResult", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__cmp__": newBuiltinFunction("__cmp__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("foo").ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ // Test `__cmp__` less than. {args: wrapArgs(newObject(cmpLtType), None), want: NewInt(-1).ToObject()}, {args: wrapArgs(None, newObject(cmpGtType)), want: NewInt(-1).ToObject()}, // Test `__cmp__` equals. {args: wrapArgs(newObject(cmpEqType), None), want: NewInt(0).ToObject()}, {args: wrapArgs(None, newObject(cmpEqType)), want: NewInt(0).ToObject()}, // Test `__cmp__` greater than. {args: wrapArgs(newObject(cmpGtType), None), want: NewInt(1).ToObject()}, {args: wrapArgs(None, newObject(cmpLtType)), want: NewInt(1).ToObject()}, // Test `__cmp__` fallback to rich comparison. {args: wrapArgs(newObject(cmpByEqType), None), want: NewInt(0).ToObject()}, {args: wrapArgs(None, newObject(cmpByEqType)), want: NewInt(0).ToObject()}, // Test bad `__cmp__` fallback to rich comparison. {args: wrapArgs(newObject(badCmpByEqType), None), wantExc: mustCreateException(TypeErrorType, "uh oh")}, {args: wrapArgs(None, newObject(badCmpByEqType)), wantExc: mustCreateException(TypeErrorType, "uh oh")}, // Test bad `__cmp__` fallback to rich comparison where a bad object is returned from `__eq__`. {args: wrapArgs(newObject(worseCmpByEqType), None), wantExc: mustCreateException(TypeErrorType, "uh oh")}, {args: wrapArgs(None, newObject(worseCmpByEqType)), wantExc: mustCreateException(TypeErrorType, "uh oh")}, // Test bad `__cmp__`. {args: wrapArgs(newObject(badCmpType), None), wantExc: mustCreateException(TypeErrorType, "uh oh")}, {args: wrapArgs(None, newObject(badCmpType)), wantExc: mustCreateException(TypeErrorType, "uh oh")}, // Test bad `__cmp__` with non-int result. {args: wrapArgs(newObject(cmpNonIntResultType), None), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, {args: wrapArgs(None, newObject(cmpNonIntResultType)), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Compare), &cas); err != "" { t.Error(err) } } } func TestCompareDefault(t *testing.T) { o1, o2 := newObject(ObjectType), newObject(ObjectType) // Make sure uintptr(o1) < uintptr(o2). if uintptr(o1.toPointer()) > uintptr(o2.toPointer()) { o1, o2 = o2, o1 } // When type names are equal, comparison should fall back to comparing // the pointer values of the types of the objects. fakeObjectType := newTestClass("object", []*Type{ObjectType}, NewDict()) o3, o4 := newObject(fakeObjectType), newObject(ObjectType) if uintptr(o3.typ.toPointer()) > uintptr(o4.typ.toPointer()) { o3, o4 = o4, o3 } // An int subtype that equals anything, but doesn't override other // comparison methods. eqType := newTestClass("Eq", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return True.ToObject(), nil }).ToObject(), "__repr__": newBuiltinFunction("__repr__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("").ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(true, o1), want: compareAllResultLT}, {args: wrapArgs(o1, -306), want: compareAllResultGT}, {args: wrapArgs(-306, o1), want: compareAllResultLT}, {args: wrapArgs(NewList(), None), want: compareAllResultGT}, {args: wrapArgs(None, "foo"), want: compareAllResultLT}, {args: wrapArgs(o1, o1), want: compareAllResultEq}, {args: wrapArgs(o1, o2), want: compareAllResultLT}, {args: wrapArgs(o2, o1), want: compareAllResultGT}, {args: wrapArgs(o3, o4), want: compareAllResultLT}, {args: wrapArgs(o4, o3), want: compareAllResultGT}, // The equality test should dispatch to the eqType instance and // return true. {args: wrapArgs(42, newObject(eqType)), want: newTestTuple(false, false, true, true, true, true).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestContains(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewTuple(), 42), want: False.ToObject()}, {args: wrapArgs(newTestList("foo", "bar"), "bar"), want: True.ToObject()}, {args: wrapArgs(newTestDict(1, "foo", 2, "bar", 3, "baz"), 2), want: True.ToObject()}, {args: wrapArgs("foobar", "ooba"), want: True.ToObject()}, {args: wrapArgs("qux", "ooba"), want: False.ToObject()}, {args: wrapArgs(3.14, None), wantExc: mustCreateException(TypeErrorType, "'float' object is not iterable")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Contains), &cas); err != "" { t.Error(err) } } } // DelAttr is tested in TestObjectDelAttr. func TestDelItem(t *testing.T) { delItem := newBuiltinFunction("TestDelItem", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestDelItem", args, ObjectType, ObjectType); raised != nil { return nil, raised } o := args[0] if raised := DelItem(f, o, args[1]); raised != nil { return nil, raised } return args[0], nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(newTestDict("foo", None), "foo"), want: NewDict().ToObject()}, {args: wrapArgs(NewDict(), "foo"), wantExc: mustCreateException(KeyErrorType, "foo")}, {args: wrapArgs(123, "bar"), wantExc: mustCreateException(TypeErrorType, "'int' object does not support item deletion")}, } for _, cas := range cases { if err := runInvokeTestCase(delItem, &cas); err != "" { t.Error(err) } } } func TestFormatException(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, t *Type, args ...*Object) (string, *BaseException) { e, raised := t.Call(f, args, nil) if raised != nil { return "", raised } f.Raise(e, nil, nil) s := FormatExc(f) f.RestoreExc(nil, nil) return s, nil }) cases := []invokeTestCase{ {args: wrapArgs(ExceptionType), want: NewStr("Exception\n").ToObject()}, {args: wrapArgs(AttributeErrorType, ""), want: NewStr("AttributeError\n").ToObject()}, {args: wrapArgs(TypeErrorType, 123), want: NewStr("TypeError: 123\n").ToObject()}, {args: wrapArgs(AttributeErrorType, "hello", "there"), want: NewStr("AttributeError: ('hello', 'there')\n").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestGetAttr(t *testing.T) { getAttr := newBuiltinFunction("TestGetAttr", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{ObjectType, StrType, ObjectType} argc := len(args) if argc == 2 { expectedTypes = expectedTypes[:2] } if raised := checkFunctionArgs(f, "TestGetAttr", args, expectedTypes...); raised != nil { return nil, raised } var def *Object if argc > 2 { def = args[2] } s, raised := ToStr(f, args[1]) if raised != nil { return nil, raised } return GetAttr(f, args[0], s, def) }).ToObject() fooResult := newObject(ObjectType) fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__getattribute__": newBuiltinFunction("__getattribute__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return fooResult, nil }).ToObject(), })) barType := newTestClass("Bar", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__getattribute__": newBuiltinFunction("__getattribute__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, "uh oh") }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(newObject(fooType), "bar"), want: fooResult}, {args: wrapArgs(newObject(fooType), "baz", None), want: fooResult}, {args: wrapArgs(newObject(ObjectType), "qux", None), want: None}, {args: wrapArgs(NewTuple(), "noexist"), wantExc: mustCreateException(AttributeErrorType, "'tuple' object has no attribute 'noexist'")}, {args: wrapArgs(DictType, "noexist"), wantExc: mustCreateException(AttributeErrorType, "type object 'dict' has no attribute 'noexist'")}, {args: wrapArgs(newObject(barType), "noexist"), wantExc: mustCreateException(TypeErrorType, "uh oh")}, } for _, cas := range cases { if err := runInvokeTestCase(getAttr, &cas); err != "" { t.Error(err) } } } func TestGetItem(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newStringDict(map[string]*Object{"foo": None}), "foo"), want: None}, {args: wrapArgs(NewDict(), "bar"), wantExc: mustCreateException(KeyErrorType, "bar")}, {args: wrapArgs(true, "baz"), wantExc: mustCreateException(TypeErrorType, "'bool' object has no attribute '__getitem__'")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(GetItem), &cas); err != "" { t.Error(err) } } } func TestHash(t *testing.T) { badHash := newTestClass("badHash", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__hash__": newBuiltinFunction("__hash__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[0], nil }).ToObject(), })) o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs("foo"), want: hashFoo}, {args: wrapArgs(123), want: NewInt(123).ToObject()}, {args: wrapArgs(o), want: NewInt(int(uintptr(o.toPointer()))).ToObject()}, {args: wrapArgs(NewList()), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, {args: wrapArgs(NewDict()), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'dict'")}, {args: wrapArgs(newObject(badHash)), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Hash), &cas); err != "" { t.Error(err) } } } func TestHex(t *testing.T) { badHex := newTestClass("badHex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__hex__": newBuiltinFunction("__hex__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewInt(123).ToObject(), nil }).ToObject(), })) goodHex := newTestClass("goodHex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__hex__": newBuiltinFunction("__hex__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewStr("0x123").ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(-123), want: NewStr("-0x7b").ToObject()}, {args: wrapArgs(123), want: NewStr("0x7b").ToObject()}, {args: wrapArgs(newObject(goodHex)), want: NewStr("0x123").ToObject()}, {args: wrapArgs(NewList()), wantExc: mustCreateException(TypeErrorType, "hex() argument can't be converted to hex")}, {args: wrapArgs(NewDict()), wantExc: mustCreateException(TypeErrorType, "hex() argument can't be converted to hex")}, {args: wrapArgs(newObject(badHex)), wantExc: mustCreateException(TypeErrorType, "__hex__ returned non-string (type int)")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Hex), &cas); err != "" { t.Error(err) } } } func TestIndex(t *testing.T) { goodType := newTestClass("GoodIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(123).ToObject(), nil }).ToObject(), })) longType := newTestClass("LongIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewLong(big.NewInt(123)).ToObject(), nil }).ToObject(), })) raiseType := newTestClass("RaiseIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(RuntimeErrorType, "uh oh") }).ToObject(), })) badType := newTestClass("BadIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewFloat(3.14).ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(42), want: NewInt(42).ToObject()}, {args: wrapArgs(newObject(goodType)), want: NewInt(123).ToObject()}, {args: wrapArgs(newObject(longType)), want: NewLong(big.NewInt(123)).ToObject()}, {args: wrapArgs(newObject(raiseType)), wantExc: mustCreateException(RuntimeErrorType, "uh oh")}, {args: wrapArgs(newObject(badType)), wantExc: mustCreateException(TypeErrorType, "__index__ returned non-(int,long) (type float)")}, {args: wrapArgs("abc"), want: None}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Index), &cas); err != "" { t.Error(err) } } cases = []invokeTestCase{ {args: wrapArgs(42), want: NewInt(42).ToObject()}, {args: wrapArgs(newObject(goodType)), want: NewInt(123).ToObject()}, {args: wrapArgs(newObject(raiseType)), wantExc: mustCreateException(RuntimeErrorType, "uh oh")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(cas.args[0].typ, "__index__", &cas); err != "" { t.Error(err) } } } func TestInvert(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(42), want: NewInt(-43).ToObject()}, {args: wrapArgs(0), want: NewInt(-1).ToObject()}, {args: wrapArgs(-35935), want: NewInt(35934).ToObject()}, {args: wrapArgs("foo"), wantExc: mustCreateException(TypeErrorType, "bad operand type for unary ~: 'str'")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Invert), &cas); err != "" { t.Error(err) } } } func TestIsInstanceIsSubclass(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, NewDict()) barType := newTestClass("Bar", []*Type{fooType, IntType}, NewDict()) cases := []struct { o *Object classinfo *Object want *Object wantExc *BaseException }{ {newObject(ObjectType), ObjectType.ToObject(), True.ToObject(), nil}, {NewInt(42).ToObject(), StrType.ToObject(), False.ToObject(), nil}, {None, NewTuple(NoneType.ToObject(), IntType.ToObject()).ToObject(), True.ToObject(), nil}, {NewStr("foo").ToObject(), NewTuple(NoneType.ToObject(), IntType.ToObject()).ToObject(), False.ToObject(), nil}, {NewStr("foo").ToObject(), NewTuple(IntType.ToObject(), NoneType.ToObject()).ToObject(), False.ToObject(), nil}, {None, NewTuple().ToObject(), False.ToObject(), nil}, {newObject(barType), fooType.ToObject(), True.ToObject(), nil}, {newObject(barType), IntType.ToObject(), True.ToObject(), nil}, {newObject(fooType), IntType.ToObject(), False.ToObject(), nil}, {newObject(ObjectType), None, nil, mustCreateException(TypeErrorType, "classinfo must be a type or tuple of types")}, {newObject(ObjectType), NewTuple(None).ToObject(), nil, mustCreateException(TypeErrorType, "classinfo must be a type or tuple of types")}, } for _, cas := range cases { // IsInstance testCase := invokeTestCase{args: wrapArgs(cas.o, cas.classinfo), want: cas.want, wantExc: cas.wantExc} if err := runInvokeTestCase(wrapFuncForTest(IsInstance), &testCase); err != "" { t.Error(err) } // IsSubclass testCase.args = wrapArgs(cas.o.Type(), cas.classinfo) if err := runInvokeTestCase(wrapFuncForTest(IsSubclass), &testCase); err != "" { t.Error(err) } } // Test that IsSubclass raises when first arg is not a type. testCase := invokeTestCase{args: wrapArgs(None, NoneType), wantExc: mustCreateException(TypeErrorType, "issubclass() arg 1 must be a class")} if err := runInvokeTestCase(wrapFuncForTest(IsSubclass), &testCase); err != "" { t.Error(err) } } func TestIsTrue(t *testing.T) { badNonZeroType := newTestClass("BadNonZeroType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__nonzero__": newBuiltinFunction("__nonzero__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return None, nil }).ToObject(), })) badLenType := newTestClass("BadLen", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__len__": newBuiltinFunction("__len__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return None, nil }).ToObject(), })) cases := []invokeTestCase{ // Bool {args: wrapArgs(true), want: True.ToObject()}, {args: wrapArgs(false), want: False.ToObject()}, // Dict {args: wrapArgs(NewDict()), want: False.ToObject()}, {args: wrapArgs(newStringDict(map[string]*Object{"foo": True.ToObject()})), want: True.ToObject()}, // Int {args: wrapArgs(0), want: False.ToObject()}, {args: wrapArgs(-1020), want: True.ToObject()}, {args: wrapArgs(1698391283), want: True.ToObject()}, // None {args: wrapArgs(None), want: False.ToObject()}, // Object {args: wrapArgs(newObject(ObjectType)), want: True.ToObject()}, // Str {args: wrapArgs(""), want: False.ToObject()}, {args: wrapArgs("\x00"), want: True.ToObject()}, {args: wrapArgs("foo"), want: True.ToObject()}, // Tuple {args: wrapArgs(NewTuple()), want: False.ToObject()}, {args: wrapArgs(newTestTuple("foo", None)), want: True.ToObject()}, // Funky types {args: wrapArgs(newObject(badNonZeroType)), wantExc: mustCreateException(TypeErrorType, "__nonzero__ should return bool, returned NoneType")}, {args: wrapArgs(newObject(badLenType)), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(IsTrue), &cas); err != "" { t.Error(err) } } } func TestIter(t *testing.T) { fun := newBuiltinFunction("TestIter", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if argc := len(args); argc != 1 { return nil, f.RaiseType(SystemErrorType, fmt.Sprintf("Iter expected 1 arg, got %d", argc)) } i, raised := Iter(f, args[0]) if raised != nil { return nil, raised } return Next(f, i) }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(NewTuple()), wantExc: mustCreateException(StopIterationType, "")}, {args: wrapArgs(newTestTuple(42, "foo")), want: NewInt(42).ToObject()}, {args: wrapArgs(newTestList("foo")), want: NewStr("foo").ToObject()}, {args: wrapArgs("foo"), want: NewStr("f").ToObject()}, {args: wrapArgs(123), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestNeg(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(42), want: NewInt(-42).ToObject()}, {args: wrapArgs(1.2), want: NewFloat(-1.2).ToObject()}, {args: wrapArgs(NewLong(big.NewInt(123))), want: NewLong(big.NewInt(-123)).ToObject()}, {args: wrapArgs("foo"), wantExc: mustCreateException(TypeErrorType, "bad operand type for unary -: 'str'")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Neg), &cas); err != "" { t.Error(err) } } } func TestNext(t *testing.T) { fun := newBuiltinFunction("TestNext", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if argc := len(args); argc != 1 { return nil, f.RaiseType(SystemErrorType, fmt.Sprintf("Next expected 1 arg, got %d", argc)) } iter := args[0] var elems []*Object elem, raised := Next(f, iter) for ; raised == nil; elem, raised = Next(f, iter) { elems = append(elems, elem) } if !raised.isInstance(StopIterationType) { return nil, raised } f.RestoreExc(nil, nil) return NewTuple(elems...).ToObject(), nil }).ToObject() testElems := []*Object{NewInt(42).ToObject(), NewStr("foo").ToObject(), newObject(ObjectType)} cases := []invokeTestCase{ {args: wrapArgs(mustNotRaise(Iter(NewRootFrame(), NewTuple().ToObject()))), want: NewTuple().ToObject()}, {args: wrapArgs(mustNotRaise(Iter(NewRootFrame(), NewTuple(testElems...).ToObject()))), want: NewTuple(testElems...).ToObject()}, {args: wrapArgs(mustNotRaise(Iter(NewRootFrame(), NewList(testElems...).ToObject()))), want: NewTuple(testElems...).ToObject()}, {args: wrapArgs(123), wantExc: mustCreateException(TypeErrorType, "int object is not an iterator")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestLen(t *testing.T) { badLenType := newTestClass("BadLen", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__len__": newBuiltinFunction("__len__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return None, nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(NewDict()), want: NewInt(0).ToObject()}, {args: wrapArgs(newStringDict(map[string]*Object{"foo": NewStr("foo value").ToObject(), "bar": NewStr("bar value").ToObject()})), want: NewInt(2).ToObject()}, {args: wrapArgs(NewTuple()), want: NewInt(0).ToObject()}, {args: wrapArgs(NewTuple(None, None, None)), want: NewInt(3).ToObject()}, {args: wrapArgs(10), wantExc: mustCreateException(TypeErrorType, "object of type 'int' has no len()")}, {args: wrapArgs(newObject(badLenType)), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Len), &cas); err != "" { t.Error(err) } } } func TestLenRaise(t *testing.T) { testTypes := []*Type{ DictType, TupleType, } for _, typ := range testTypes { cases := []invokeTestCase{ {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, fmt.Sprintf("unbound method __len__() must be called with %s instance as first argument (got nothing instead)", typ.Name()))}, {args: wrapArgs(newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, fmt.Sprintf("unbound method __len__() must be called with %s instance as first argument (got object instance instead)", typ.Name()))}, {args: wrapArgs(newObject(ObjectType), newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, fmt.Sprintf("unbound method __len__() must be called with %s instance as first argument (got object instance instead)", typ.Name()))}, } for _, cas := range cases { if err := runInvokeMethodTestCase(typ, "__len__", &cas); err != "" { t.Error(err) } } } } func TestInvokePositionalArgs(t *testing.T) { fun := newBuiltinFunction("TestInvokePositionalArgs", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return NewTuple(args.makeCopy()...).ToObject(), nil }).ToObject() cases := []struct { varargs *Object args Args want *Object }{ {nil, nil, NewTuple().ToObject()}, {NewTuple(NewInt(2).ToObject()).ToObject(), nil, NewTuple(NewInt(2).ToObject()).ToObject()}, {nil, []*Object{NewStr("foo").ToObject()}, NewTuple(NewStr("foo").ToObject()).ToObject()}, {NewTuple(NewFloat(3.14).ToObject()).ToObject(), []*Object{NewStr("foo").ToObject()}, NewTuple(NewStr("foo").ToObject(), NewFloat(3.14).ToObject()).ToObject()}, {NewList(NewFloat(3.14).ToObject()).ToObject(), []*Object{NewStr("foo").ToObject()}, NewTuple(NewStr("foo").ToObject(), NewFloat(3.14).ToObject()).ToObject()}, } for _, cas := range cases { got, raised := Invoke(NewRootFrame(), fun, cas.args, cas.varargs, nil, nil) switch checkResult(got, cas.want, raised, nil) { case checkInvokeResultExceptionMismatch: t.Errorf("PackArgs(%v, %v) raised %v, want nil", cas.args, cas.varargs, raised) case checkInvokeResultReturnValueMismatch: t.Errorf("PackArgs(%v, %v) = %v, want %v", cas.args, cas.varargs, got, cas.want) } } } func TestInvokeKeywordArgs(t *testing.T) { fun := newBuiltinFunction("TestInvokeKeywordArgs", func(f *Frame, _ Args, kwargs KWArgs) (*Object, *BaseException) { got := map[string]*Object{} for _, kw := range kwargs { got[kw.Name] = kw.Value } return newStringDict(got).ToObject(), nil }).ToObject() d := NewDict() d.SetItem(NewRootFrame(), NewInt(123).ToObject(), None) cases := []struct { keywords KWArgs kwargs *Object want *Object wantExc *BaseException }{ {nil, nil, NewDict().ToObject(), nil}, {wrapKWArgs("foo", 42), nil, newTestDict("foo", 42).ToObject(), nil}, {nil, newTestDict("foo", None).ToObject(), newTestDict("foo", None).ToObject(), nil}, {wrapKWArgs("foo", 42), newTestDict("bar", None).ToObject(), newTestDict("foo", 42, "bar", None).ToObject(), nil}, {nil, NewList().ToObject(), nil, mustCreateException(TypeErrorType, "argument after ** must be a dict, not list")}, {nil, d.ToObject(), nil, mustCreateException(TypeErrorType, "keywords must be strings")}, } for _, cas := range cases { got, raised := Invoke(NewRootFrame(), fun, nil, nil, cas.keywords, cas.kwargs) switch checkResult(got, cas.want, raised, cas.wantExc) { case checkInvokeResultExceptionMismatch: t.Errorf("PackKwargs(%v, %v) raised %v, want %v", cas.keywords, cas.kwargs, raised, cas.wantExc) case checkInvokeResultReturnValueMismatch: t.Errorf("PackKwargs(%v, %v) = %v, want %v", cas.keywords, cas.kwargs, got, cas.want) } } } func TestOct(t *testing.T) { badOct := newTestClass("badOct", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__oct__": newBuiltinFunction("__oct__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewInt(123).ToObject(), nil }).ToObject(), })) goodOct := newTestClass("goodOct", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__oct__": newBuiltinFunction("__oct__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewStr("0123").ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(-123), want: NewStr("-0173").ToObject()}, {args: wrapArgs(123), want: NewStr("0173").ToObject()}, {args: wrapArgs(newObject(goodOct)), want: NewStr("0123").ToObject()}, {args: wrapArgs(NewList()), wantExc: mustCreateException(TypeErrorType, "oct() argument can't be converted to oct")}, {args: wrapArgs(NewDict()), wantExc: mustCreateException(TypeErrorType, "oct() argument can't be converted to oct")}, {args: wrapArgs(newObject(badOct)), wantExc: mustCreateException(TypeErrorType, "__oct__ returned non-string (type int)")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Oct), &cas); err != "" { t.Error(err) } } } func TestPos(t *testing.T) { pos := newTestClass("pos", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__pos__": newBuiltinFunction("__pos__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(-42).ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(42), want: NewInt(42).ToObject()}, {args: wrapArgs(1.2), want: NewFloat(1.2).ToObject()}, {args: wrapArgs(NewLong(big.NewInt(123))), want: NewLong(big.NewInt(123)).ToObject()}, {args: wrapArgs(newObject(pos)), want: NewInt(-42).ToObject()}, {args: wrapArgs("foo"), wantExc: mustCreateException(TypeErrorType, "bad operand type for unary +: 'str'")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Pos), &cas); err != "" { t.Error(err) } } } func TestPyPrint(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, args *Tuple, sep, end string) (string, *BaseException) { return captureStdout(f, func() *BaseException { return pyPrint(NewRootFrame(), args.elems, sep, end, Stdout) }) }) cases := []invokeTestCase{ {args: wrapArgs(NewTuple(), "", "\n"), want: NewStr("\n").ToObject()}, {args: wrapArgs(NewTuple(), "", ""), want: NewStr("").ToObject()}, {args: wrapArgs(newTestTuple("abc", 123), " ", "\n"), want: NewStr("abc 123\n").ToObject()}, {args: wrapArgs(newTestTuple("foo"), "", " "), want: NewStr("foo ").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } // TODO(corona10): Re-enable once #282 is addressed. /*func TestPrint(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, args *Tuple, nl bool) (string, *BaseException) { return captureStdout(f, func() *BaseException { return Print(NewRootFrame(), args.elems, nl) }) }) cases := []invokeTestCase{ {args: wrapArgs(NewTuple(), true), want: NewStr("\n").ToObject()}, {args: wrapArgs(NewTuple(), false), want: NewStr("").ToObject()}, {args: wrapArgs(newTestTuple("abc", 123), true), want: NewStr("abc 123\n").ToObject()}, {args: wrapArgs(newTestTuple("foo"), false), want: NewStr("foo ").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } }*/ func TestReprRaise(t *testing.T) { testTypes := []*Type{ BaseExceptionType, BoolType, DictType, IntType, FunctionType, StrType, TupleType, TypeType, } for _, typ := range testTypes { cases := []invokeTestCase{ {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, fmt.Sprintf("unbound method __repr__() must be called with %s instance as first argument (got nothing instead)", typ.Name()))}, {args: wrapArgs(newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, fmt.Sprintf("unbound method __repr__() must be called with %s instance as first argument (got object instance instead)", typ.Name()))}, } for _, cas := range cases { if err := runInvokeMethodTestCase(typ, "__repr__", &cas); err != "" { t.Error(err) } } } } func TestReprMethodReturnsNonStr(t *testing.T) { // Don't use runInvokeTestCase since it takes repr(args) and in this // case repr will raise. typ := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__repr__": newBuiltinFunction("__repr__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return None, nil }).ToObject(), })) _, raised := Repr(NewRootFrame(), newObject(typ)) wantExc := mustCreateException(TypeErrorType, "__repr__ returned non-string (type NoneType)") if !exceptionsAreEquivalent(raised, wantExc) { t.Errorf(`Repr() raised %v, want %v`, raised, wantExc) } } func TestResolveClass(t *testing.T) { f := NewRootFrame() cases := []struct { class *Dict local *Object globals *Dict name string want *Object wantExc *BaseException }{ {newStringDict(map[string]*Object{"foo": NewStr("bar").ToObject()}), NewStr("baz").ToObject(), NewDict(), "foo", NewStr("bar").ToObject(), nil}, {newStringDict(map[string]*Object{"str": NewInt(42).ToObject()}), nil, NewDict(), "str", NewInt(42).ToObject(), nil}, {NewDict(), nil, newStringDict(map[string]*Object{"foo": NewStr("bar").ToObject()}), "foo", NewStr("bar").ToObject(), nil}, {NewDict(), nil, NewDict(), "str", StrType.ToObject(), nil}, {NewDict(), nil, NewDict(), "foo", nil, mustCreateException(NameErrorType, "name 'foo' is not defined")}, } for _, cas := range cases { f.globals = cas.globals got, raised := ResolveClass(f, cas.class, cas.local, NewStr(cas.name)) switch checkResult(got, cas.want, raised, cas.wantExc) { case checkInvokeResultExceptionMismatch: t.Errorf("ResolveClass(%v, %q) raised %v, want %v", cas.globals, cas.name, raised, cas.wantExc) case checkInvokeResultReturnValueMismatch: t.Errorf("ResolveClass(%v, %q) = %v, want %v", cas.globals, cas.name, got, cas.want) } } } func TestResolveGlobal(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, globals *Dict, name *Str) (*Object, *BaseException) { f.globals = globals return ResolveGlobal(f, name) }) cases := []invokeTestCase{ {args: wrapArgs(newStringDict(map[string]*Object{"foo": NewStr("bar").ToObject()}), "foo"), want: NewStr("bar").ToObject()}, {args: wrapArgs(NewDict(), "str"), want: StrType.ToObject()}, {args: wrapArgs(NewDict(), "foo"), wantExc: mustCreateException(NameErrorType, "name 'foo' is not defined")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestRichCompare(t *testing.T) { badCmpType := newTestClass("BadCmp", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__cmp__": newBuiltinFunction("__cmp__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, "uh oh") }).ToObject(), })) cmpEqType := newTestClass("BadCmp", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__cmp__": newBuiltinFunction("__cmp__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewInt(0).ToObject(), nil }).ToObject(), })) cmpByEqType := newTestClass("Eq", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return True.ToObject(), nil }).ToObject(), "__cmp__": newBuiltinFunction("__cmp__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NotImplemented, nil }).ToObject(), })) badCmpEqType := newTestClass("Eq", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, "uh oh") }).ToObject(), })) cases := []invokeTestCase{ // Test `__eq__` fallback to `__cmp__`. {args: wrapArgs(newObject(cmpEqType), newObject(cmpEqType)), want: compareAllResultEq}, // Test `__cmp__` fallback to `__eq__`. {args: wrapArgs(newObject(cmpByEqType), newObject(cmpByEqType)), want: compareAllResultEq}, // Test rich compare fallback to bad `__cmp__`. {args: wrapArgs(newObject(badCmpType), newObject(badCmpType)), wantExc: mustCreateException(TypeErrorType, "uh oh")}, // Test bad `__eq__` where the second object being compared is a subclass of the first. {args: wrapArgs(NewInt(13).ToObject(), newObject(badCmpEqType)), wantExc: mustCreateException(TypeErrorType, "uh oh")}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestCheckLocal(t *testing.T) { o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(o, "foo"), want: None}, {args: wrapArgs(UnboundLocal, "bar"), wantExc: mustCreateException(UnboundLocalErrorType, "local variable 'bar' referenced before assignment")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(CheckLocal), &cas); err != "" { t.Error(err) } } } func TestSetItem(t *testing.T) { setItem := newBuiltinFunction("TestSetItem", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestSetItem", args, ObjectType, ObjectType, ObjectType); raised != nil { return nil, raised } o := args[0] if raised := SetItem(f, o, args[1], args[2]); raised != nil { return nil, raised } return o, nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(NewDict(), "bar", None), want: newTestDict("bar", None).ToObject()}, {args: wrapArgs(123, "bar", None), wantExc: mustCreateException(TypeErrorType, "'int' object has no attribute '__setitem__'")}, } for _, cas := range cases { if err := runInvokeTestCase(setItem, &cas); err != "" { t.Error(err) } } } func TestStartThread(t *testing.T) { c := make(chan bool) callable := newBuiltinFunction("TestStartThread", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { close(c) return None, nil }).ToObject() StartThread(callable) // Deadlock indicates the thread didn't start. <-c } func TestStartThreadRaises(t *testing.T) { // Since there's no way to notify that the goroutine has returned we // can't actually test the exception output but we can at least make // sure the callable ran and didn't blow up the rest of the program. c := make(chan bool) callable := newBuiltinFunction("TestStartThreadRaises", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { defer close(c) return nil, f.RaiseType(ExceptionType, "foo") }).ToObject() StartThread(callable) <-c } func TestTie(t *testing.T) { targets := make([]*Object, 3) cases := []struct { t TieTarget o *Object want *Object wantExc *BaseException }{ {TieTarget{Target: &targets[0]}, NewInt(42).ToObject(), NewTuple(NewInt(42).ToObject()).ToObject(), nil}, {TieTarget{Target: &targets[0]}, NewTuple().ToObject(), NewTuple(NewTuple().ToObject()).ToObject(), nil}, { TieTarget{ Children: []TieTarget{{Target: &targets[0]}, {Target: &targets[1]}}, }, NewList(NewStr("foo").ToObject(), NewStr("bar").ToObject()).ToObject(), NewTuple(NewStr("foo").ToObject(), NewStr("bar").ToObject()).ToObject(), nil, }, { TieTarget{ Children: []TieTarget{ {Target: &targets[0]}, {Children: []TieTarget{{Target: &targets[1]}, {Target: &targets[2]}}}, }, }, NewTuple(NewStr("foo").ToObject(), NewTuple(NewStr("bar").ToObject(), NewStr("baz").ToObject()).ToObject()).ToObject(), NewTuple(NewStr("foo").ToObject(), NewStr("bar").ToObject(), NewStr("baz").ToObject()).ToObject(), nil, }, { TieTarget{ Children: []TieTarget{ {Target: &targets[0]}, {Target: &targets[1]}, }, }, NewList(NewStr("foo").ToObject()).ToObject(), nil, mustCreateException(ValueErrorType, "need more than 1 values to unpack"), }, { TieTarget{Children: []TieTarget{{Target: &targets[0]}}}, NewTuple(NewInt(1).ToObject(), NewInt(2).ToObject()).ToObject(), nil, mustCreateException(ValueErrorType, "too many values to unpack"), }, } for _, cas := range cases { for i := range targets { targets[i] = nil } var got *Object raised := Tie(NewRootFrame(), cas.t, cas.o) if raised == nil { var elems []*Object for _, t := range targets { if t == nil { break } elems = append(elems, t) } got = NewTuple(elems...).ToObject() } switch checkResult(got, cas.want, raised, cas.wantExc) { case checkInvokeResultExceptionMismatch: t.Errorf("Tie(%+v, %v) raised %v, want %v", cas.t, cas.o, raised, cas.wantExc) case checkInvokeResultReturnValueMismatch: t.Errorf("Tie(%+v, %v) = %v, want %v", cas.t, cas.o, got, cas.want) } } } func TestToInt(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object) (*Tuple, *BaseException) { i, raised := ToInt(f, o) if raised != nil { return nil, raised } return newTestTuple(i, i.Type()), nil }) cases := []invokeTestCase{ {args: wrapArgs(42), want: newTestTuple(42, IntType).ToObject()}, {args: wrapArgs(big.NewInt(123)), want: newTestTuple(123, LongType).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestToIntValue(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(42), want: NewInt(42).ToObject()}, {args: wrapArgs(big.NewInt(123)), want: NewInt(123).ToObject()}, {args: wrapArgs(overflowLong), wantExc: mustCreateException(OverflowErrorType, "Python int too large to convert to a Go int")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToIntValue), &cas); err != "" { t.Error(err) } } } func TestToNative(t *testing.T) { foo := newObject(ObjectType) cases := []struct { o *Object want interface{} wantExc *BaseException }{ {True.ToObject(), true, nil}, {NewInt(42).ToObject(), 42, nil}, {NewStr("bar").ToObject(), "bar", nil}, {foo, foo, nil}, } for _, cas := range cases { got, raised := ToNative(NewRootFrame(), cas.o) if !exceptionsAreEquivalent(raised, cas.wantExc) { t.Errorf("ToNative(%v) raised %v, want %v", cas.o, raised, cas.wantExc) } else if raised == nil && (!got.IsValid() || !reflect.DeepEqual(got.Interface(), cas.want)) { t.Errorf("ToNative(%v) = %v, want %v", cas.o, got, cas.want) } } } func BenchmarkGetAttr(b *testing.B) { f := NewRootFrame() attr := NewStr("bar") fooType := newTestClass("Foo", []*Type{ObjectType}, NewDict()) foo := newObject(fooType) if raised := SetAttr(f, foo, attr, NewInt(123).ToObject()); raised != nil { panic(raised) } b.ResetTimer() for i := 0; i < b.N; i++ { mustNotRaise(GetAttr(f, foo, attr, nil)) } } // SetAttr is tested in TestObjectSetAttr. func exceptionsAreEquivalent(e1 *BaseException, e2 *BaseException) bool { if e1 == nil && e2 == nil { return true } if e1 == nil || e2 == nil { return false } if e1.typ != e2.typ { return false } if e1.args == nil && e2.args == nil { return true } if e1.args == nil || e2.args == nil { return false } f := NewRootFrame() b, raised := IsTrue(f, mustNotRaise(Eq(f, e1.args.ToObject(), e2.args.ToObject()))) if raised != nil { panic(raised) } return b } func getFuncName(f interface{}) string { s := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() return regexp.MustCompile(`\w+$`).FindString(s) } // wrapFuncForTest creates a callable object that invokes fun, passing the // current frame as its first argument followed by caller provided args. func wrapFuncForTest(fun interface{}) *Object { return newBuiltinFunction(getFuncName(fun), func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { callable, raised := WrapNative(f, reflect.ValueOf(fun)) if raised != nil { return nil, raised } argc := len(args) nativeArgs := make(Args, argc+1, argc+1) nativeArgs[0] = f.ToObject() copy(nativeArgs[1:], args) return callable.Call(f, nativeArgs, nil) }).ToObject() } func mustCreateException(t *Type, msg string) *BaseException { if !t.isSubclass(BaseExceptionType) { panic(fmt.Sprintf("type does not inherit from BaseException: %s", t.Name())) } e := toBaseExceptionUnsafe(newObject(t)) if msg == "" { e.args = NewTuple() } else { e.args = NewTuple(NewStr(msg).ToObject()) } return e } func mustNotRaise(o *Object, raised *BaseException) *Object { if raised != nil { panic(raised) } return o } var ( compareAll = wrapFuncForTest(func(f *Frame, v, w *Object) (*Object, *BaseException) { lt, raised := LT(f, v, w) if raised != nil { return nil, raised } le, raised := LE(f, v, w) if raised != nil { return nil, raised } eq, raised := Eq(f, v, w) if raised != nil { return nil, raised } ne, raised := NE(f, v, w) if raised != nil { return nil, raised } ge, raised := GE(f, v, w) if raised != nil { return nil, raised } gt, raised := GT(f, v, w) if raised != nil { return nil, raised } return NewTuple(lt, le, eq, ne, ge, gt).ToObject(), nil }) compareAllResultLT = newTestTuple(true, true, false, true, false, false).ToObject() compareAllResultEq = newTestTuple(false, true, true, false, true, false).ToObject() compareAllResultGT = newTestTuple(false, false, false, true, true, true).ToObject() ) ================================================ FILE: runtime/descriptor.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" ) type fieldDescriptorType int const ( fieldDescriptorRO fieldDescriptorType = iota fieldDescriptorRW ) // Property represents Python 'property' objects. type Property struct { Object get, set, del *Object } func newProperty(get, set, del *Object) *Property { return &Property{Object{typ: PropertyType}, get, set, del} } func toPropertyUnsafe(o *Object) *Property { return (*Property)(o.toPointer()) } // ToObject upcasts p to an Object. func (p *Property) ToObject() *Object { return &p.Object } // PropertyType is the object representing the Python 'property' type. var PropertyType = newBasisType("property", reflect.TypeOf(Property{}), toPropertyUnsafe, ObjectType) func initPropertyType(map[string]*Object) { PropertyType.slots.Delete = &deleteSlot{propertyDelete} PropertyType.slots.Get = &getSlot{propertyGet} PropertyType.slots.Init = &initSlot{propertyInit} PropertyType.slots.Set = &setSlot{propertySet} } func propertyDelete(f *Frame, desc, inst *Object) *BaseException { p := toPropertyUnsafe(desc) if p.del == nil || p.del == None { return f.RaiseType(AttributeErrorType, "can't delete attribute") } _, raised := p.del.Call(f, Args{inst}, nil) return raised } func propertyGet(f *Frame, desc, instance *Object, _ *Type) (*Object, *BaseException) { p := toPropertyUnsafe(desc) if p.get == nil || p.get == None { return nil, f.RaiseType(AttributeErrorType, "unreadable attribute") } return p.get.Call(f, Args{instance}, nil) } func propertyInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{ObjectType, ObjectType, ObjectType} argc := len(args) if argc < 3 { expectedTypes = expectedTypes[:argc] } if raised := checkFunctionArgs(f, "__init__", args, expectedTypes...); raised != nil { return nil, raised } p := toPropertyUnsafe(o) if argc > 0 { p.get = args[0] } if argc > 1 { p.set = args[1] } if argc > 2 { p.del = args[2] } return None, nil } func propertySet(f *Frame, desc, inst, value *Object) *BaseException { p := toPropertyUnsafe(desc) if p.set == nil || p.set == None { return f.RaiseType(AttributeErrorType, "can't set attribute") } _, raised := p.set.Call(f, Args{inst, value}, nil) return raised } // makeStructFieldDescriptor creates a descriptor with a getter that returns // the field given by fieldName from t's basis structure. func makeStructFieldDescriptor(t *Type, fieldName, propertyName string, fieldMode fieldDescriptorType) *Object { field, ok := t.basis.FieldByName(fieldName) if !ok { logFatal(fmt.Sprintf("no such field %q for basis %s", fieldName, nativeTypeName(t.basis))) } getterFunc := func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, fieldName, args, ObjectType); raised != nil { return nil, raised } self := args[0] if !self.isInstance(t) { format := "descriptor '%s' for '%s' objects doesn't apply to '%s' objects" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, propertyName, t.Name(), self.typ.Name())) } return WrapNative(f, t.slots.Basis.Fn(self).FieldByIndex(field.Index)) } getter := newBuiltinFunction("_get"+fieldName, getterFunc).ToObject() setter := None if fieldMode == fieldDescriptorRW { if field.PkgPath != "" { logFatal(fmt.Sprintf("field '%q' is not public on Golang code. Please fix it.", fieldName)) } setterFunc := func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, fieldName, args, ObjectType, ObjectType); raised != nil { return nil, raised } self := args[0] newValue := args[1] if !self.isInstance(t) { format := "descriptor '%s' for '%s' objects doesn't apply to '%s' objects" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, propertyName, t.Name(), self.typ.Name())) } val := t.slots.Basis.Fn(self).FieldByIndex(field.Index) converted, raised := maybeConvertValue(f, newValue, field.Type) if raised != nil { return nil, raised } val.Set(converted) return None, nil } setter = newBuiltinFunction("_set"+fieldName, setterFunc).ToObject() } return newProperty(getter, setter, None).ToObject() } ================================================ FILE: runtime/descriptor_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestPropertyDelete(t *testing.T) { dummy := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(newProperty(nil, nil, wrapFuncForTest(func(f *Frame, o *Object) (*Object, *BaseException) { return None, nil })), dummy), want: None}, {args: wrapArgs(newProperty(nil, nil, wrapFuncForTest(func(f *Frame, o *Object) (*Object, *BaseException) { return nil, f.RaiseType(ValueErrorType, "bar") })), dummy), wantExc: mustCreateException(ValueErrorType, "bar")}, {args: wrapArgs(newProperty(nil, nil, nil), dummy), wantExc: mustCreateException(AttributeErrorType, "can't delete attribute")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(PropertyType, "__delete__", &cas); err != "" { t.Error(err) } } } func TestPropertyGet(t *testing.T) { dummy := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(newProperty(wrapFuncForTest(func(f *Frame, o *Object) (*Object, *BaseException) { return o, nil }), nil, nil), dummy, ObjectType), want: dummy}, {args: wrapArgs(newProperty(wrapFuncForTest(func(f *Frame, o *Object) (*Object, *BaseException) { return nil, f.RaiseType(ValueErrorType, "bar") }), nil, nil), dummy, ObjectType), wantExc: mustCreateException(ValueErrorType, "bar")}, {args: wrapArgs(newProperty(nil, nil, nil), dummy, ObjectType), wantExc: mustCreateException(AttributeErrorType, "unreadable attribute")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(PropertyType, "__get__", &cas); err != "" { t.Error(err) } } } func TestPropertyInit(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Object, *BaseException) { o, raised := PropertyType.Call(f, args, nil) if raised != nil { return nil, raised } p := toPropertyUnsafe(o) return newTestTuple(p.get, p.set, p.del).ToObject(), nil }) cases := []invokeTestCase{ {want: NewTuple(None, None, None).ToObject()}, {args: wrapArgs("foo"), want: newTestTuple("foo", None, None).ToObject()}, {args: wrapArgs("foo", None), want: newTestTuple("foo", None, None).ToObject()}, {args: wrapArgs("foo", None, "bar"), want: newTestTuple("foo", None, "bar").ToObject()}, {args: wrapArgs(1, 2, 3, 4), wantExc: mustCreateException(TypeErrorType, "'__init__' requires 3 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestPropertySet(t *testing.T) { dummy := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(newProperty(nil, wrapFuncForTest(func(_ *Frame, _, _ *Object) (*Object, *BaseException) { return None, nil }), nil), dummy, 123), want: None}, {args: wrapArgs(newProperty(nil, wrapFuncForTest(func(f *Frame, _, _ *Object) (*Object, *BaseException) { return nil, f.RaiseType(ValueErrorType, "bar") }), nil), dummy, 123), wantExc: mustCreateException(ValueErrorType, "bar")}, {args: wrapArgs(newProperty(nil, nil, nil), dummy, 123), wantExc: mustCreateException(AttributeErrorType, "can't set attribute")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(PropertyType, "__set__", &cas); err != "" { t.Error(err) } } } func TestMakeStructFieldDescriptor(t *testing.T) { e := mustNotRaise(RuntimeErrorType.Call(NewRootFrame(), wrapArgs("foo"), nil)) fun := newBuiltinFunction("TestMakeStructFieldDescriptor", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "TestMakeStructFieldDescriptor", args, TypeType, StrType, StrType, ObjectType); raised != nil { return nil, raised } t := toTypeUnsafe(args[0]) desc := makeStructFieldDescriptor(t, toStrUnsafe(args[1]).Value(), toStrUnsafe(args[2]).Value(), fieldDescriptorRO) get, raised := GetAttr(f, desc, NewStr("__get__"), nil) if raised != nil { return nil, raised } return get.Call(f, wrapArgs(args[3], t), nil) }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(ObjectType, "dict", "__dict__", newObject(ObjectType)), want: None}, {args: wrapArgs(ObjectType, "dict", "__dict__", newBuiltinFunction("foo", func(*Frame, Args, KWArgs) (*Object, *BaseException) { return nil, nil })), want: NewDict().ToObject()}, {args: wrapArgs(IntType, "value", "value", 42), want: NewInt(42).ToObject()}, {args: wrapArgs(StrType, "value", "value", 42), wantExc: mustCreateException(TypeErrorType, "descriptor 'value' for 'str' objects doesn't apply to 'int' objects")}, {args: wrapArgs(BaseExceptionType, "args", "args", e), want: NewTuple(NewStr("foo").ToObject()).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestMakeStructFieldDescriptorRWGet(t *testing.T) { fun := newBuiltinFunction("TestMakeStructFieldDescriptorRW_get", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "TestMakeStructFieldDescriptorRW_get", args, TypeType, StrType, StrType, ObjectType); raised != nil { return nil, raised } t := toTypeUnsafe(args[0]) desc := makeStructFieldDescriptor(t, toStrUnsafe(args[1]).Value(), toStrUnsafe(args[2]).Value(), fieldDescriptorRW) get, raised := GetAttr(f, desc, NewStr("__get__"), nil) if raised != nil { return nil, raised } return get.Call(f, wrapArgs(args[3], t), nil) }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(FileType, "Softspace", "softspace", newObject(FileType)), want: NewInt(0).ToObject()}, {args: wrapArgs(FileType, "Softspace", "softspace", 42), wantExc: mustCreateException(TypeErrorType, "descriptor 'softspace' for 'file' objects doesn't apply to 'int' objects")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestMakeStructFieldDescriptorRWSet(t *testing.T) { fun := newBuiltinFunction("TestMakeStructFieldDescriptorRW_set", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "TestMakeStructFieldDescriptorRW_set", args, TypeType, StrType, StrType, ObjectType, ObjectType); raised != nil { return nil, raised } t := toTypeUnsafe(args[0]) desc := makeStructFieldDescriptor(t, toStrUnsafe(args[1]).Value(), toStrUnsafe(args[2]).Value(), fieldDescriptorRW) set, raised := GetAttr(f, desc, NewStr("__set__"), nil) if raised != nil { return nil, raised } return set.Call(f, wrapArgs(args[3], args[4]), nil) }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(FileType, "Softspace", "softspace", newObject(FileType), NewInt(0).ToObject()), want: None}, {args: wrapArgs(FileType, "Softspace", "softspace", newObject(FileType), NewInt(0)), want: None}, {args: wrapArgs(FileType, "Softspace", "softspace", newObject(FileType), "wrong"), wantExc: mustCreateException(TypeErrorType, "an int is required")}, {args: wrapArgs(FileType, "Softspace", "softspace", 42, NewInt(0)), wantExc: mustCreateException(TypeErrorType, "descriptor 'softspace' for 'file' objects doesn't apply to 'int' objects")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/dict.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "bytes" "fmt" "reflect" "sync/atomic" "unsafe" ) var ( // DictType is the object representing the Python 'dict' type. DictType = newBasisType("dict", reflect.TypeOf(Dict{}), toDictUnsafe, ObjectType) dictItemIteratorType = newBasisType("dictionary-itemiterator", reflect.TypeOf(dictItemIterator{}), toDictItemIteratorUnsafe, ObjectType) dictKeyIteratorType = newBasisType("dictionary-keyiterator", reflect.TypeOf(dictKeyIterator{}), toDictKeyIteratorUnsafe, ObjectType) dictValueIteratorType = newBasisType("dictionary-valueiterator", reflect.TypeOf(dictValueIterator{}), toDictValueIteratorUnsafe, ObjectType) deletedEntry = &dictEntry{} ) const ( // maxDictSize is the largest number of entries a dictionary can hold. // Dict sizes must be a power of two and this is the largest such // number representable as int32. maxDictSize = 1 << 30 minDictSize = 8 ) // dictEntry represents a slot in the hash table of a Dict. Entries are // intended to be immutable so that they can be read atomically. type dictEntry struct { hash int key *Object value *Object } // dictTable is the hash table underlying Dict. type dictTable struct { // used is the number of slots in the entries table that contain values. used int32 // fill is the number of slots that are used or once were used but have // since been cleared. Thus used <= fill <= len(entries). fill int // entries is a slice of immutable dict entries. Although elements in // the slice will be modified to point to different dictEntry objects // as the dictionary is updated, the slice itself (i.e. location in // memory and size) will not change for the lifetime of a dictTable. // When the table is no longer large enough to hold a dict's contents, // a new dictTable will be created. entries []*dictEntry } // newDictTable allocates a table where at least minCapacity entries can be // accommodated. minCapacity must be <= maxDictSize. func newDictTable(minCapacity int) *dictTable { // This takes the given capacity and sets all bits less than the highest bit. // Adding 1 to that value causes the number to become a multiple of 2 again. // The minDictSize is mixed in to make sure the resulting value is at least // that big. This implementation makes the function able to be inlined, as // well as allows for complete evaluation of constants at compile time. numEntries := (minDictSize - 1) | minCapacity numEntries |= numEntries >> 1 numEntries |= numEntries >> 2 numEntries |= numEntries >> 4 numEntries |= numEntries >> 8 numEntries |= numEntries >> 16 return &dictTable{entries: make([]*dictEntry, numEntries+1)} } // loadEntry atomically loads the i'th entry in t and returns it. func (t *dictTable) loadEntry(i int) *dictEntry { p := (*unsafe.Pointer)(unsafe.Pointer(&t.entries[i])) return (*dictEntry)(atomic.LoadPointer(p)) } // storeEntry atomically sets the i'th entry in t to entry. func (t *dictTable) storeEntry(i int, entry *dictEntry) { p := (*unsafe.Pointer)(unsafe.Pointer(&t.entries[i])) atomic.StorePointer(p, unsafe.Pointer(entry)) } func (t *dictTable) loadUsed() int { return int(atomic.LoadInt32(&t.used)) } func (t *dictTable) incUsed(n int) { atomic.AddInt32(&t.used, int32(n)) } // insertAbsentEntry adds the populated entry to t assuming that the key // specified in entry is absent from t. Since the key is absent, no key // comparisons are necessary to perform the insert. func (t *dictTable) insertAbsentEntry(entry *dictEntry) { mask := uint(len(t.entries) - 1) i := uint(entry.hash) & mask perturb := uint(entry.hash) index := i // The key we're trying to insert is known to be absent from the dict // so probe for the first nil entry. for ; t.entries[index] != nil; index = i & mask { i, perturb = dictNextIndex(i, perturb) } t.entries[index] = entry t.incUsed(1) t.fill++ } // lookupEntry returns the index and entry in t with the given hash and key. // Elements in the table are updated with immutable entries atomically and // lookupEntry loads them atomically. So it is not necessary to lock the dict // to do entry lookups in a consistent way. func (t *dictTable) lookupEntry(f *Frame, hash int, key *Object) (int, *dictEntry, *BaseException) { mask := uint(len(t.entries) - 1) i, perturb := uint(hash)&mask, uint(hash) // free is the first slot that's available. We don't immediately use it // because it has been previously used and therefore an exact match may // be found further on. free := -1 var freeEntry *dictEntry index := int(i & mask) entry := t.loadEntry(index) for { if entry == nil { if free != -1 { index = free // Store the entry instead of fetching by index // later since it may have changed by then. entry = freeEntry } break } if entry == deletedEntry { if free == -1 { free = index } } else if entry.hash == hash { o, raised := Eq(f, entry.key, key) if raised != nil { return -1, nil, raised } eq, raised := IsTrue(f, o) if raised != nil { return -1, nil, raised } if eq { break } } i, perturb = dictNextIndex(i, perturb) index = int(i & mask) entry = t.loadEntry(index) } return index, entry, nil } // writeEntry replaces t's entry at the given index with entry. If writing // entry would cause t's fill ratio to grow too large then a new table is // created, the entry is instead inserted there and that table is returned. t // remains unchanged. When a sufficiently sized table cannot be created, false // will be returned for the second value, otherwise true will be returned. func (t *dictTable) writeEntry(f *Frame, index int, entry *dictEntry) (*dictTable, bool) { if t.entries[index] == deletedEntry { t.storeEntry(index, entry) t.incUsed(1) return nil, true } if t.entries[index] != nil { t.storeEntry(index, entry) return nil, true } if (t.fill+1)*3 <= len(t.entries)*2 { // New entry does not necessitate growing the table. t.storeEntry(index, entry) t.incUsed(1) t.fill++ return nil, true } // Grow the table. var n int if t.used <= 50000 { n = int(t.used * 4) } else if t.used <= maxDictSize/2 { n = int(t.used * 2) } else { return nil, false } newTable := newDictTable(n) for _, oldEntry := range t.entries { if oldEntry != nil && oldEntry != deletedEntry { newTable.insertAbsentEntry(oldEntry) } } newTable.insertAbsentEntry(entry) return newTable, true } // dictEntryIterator is used to iterate over the entries in a dictTable in an // arbitrary order. type dictEntryIterator struct { index int64 table *dictTable } // newDictEntryIterator creates a dictEntryIterator object for d. It assumes // that d.mutex is held by the caller. func newDictEntryIterator(d *Dict) dictEntryIterator { return dictEntryIterator{table: d.loadTable()} } // next advances this iterator to the next occupied entry and returns it. The // second return value is true if the dict changed since iteration began, false // otherwise. func (iter *dictEntryIterator) next() *dictEntry { numEntries := len(iter.table.entries) var entry *dictEntry for entry == nil { // 64bit atomic ops need to be 8 byte aligned. This compile time check // verifies alignment by creating a negative constant for an unsigned type. // See sync/atomic docs for details. const blank = -(unsafe.Offsetof(iter.index) % 8) index := int(atomic.AddInt64(&iter.index, 1)) - 1 if index >= numEntries { break } entry = iter.table.loadEntry(index) if entry == deletedEntry { entry = nil } } return entry } // dictVersionGuard is used to detect when a dict has been modified. type dictVersionGuard struct { dict *Dict version int64 } func newDictVersionGuard(d *Dict) dictVersionGuard { return dictVersionGuard{d, d.loadVersion()} } // check returns false if the dict held by g has changed since g was created, // true otherwise. func (g *dictVersionGuard) check() bool { return g.dict.loadVersion() == g.version } // Dict represents Python 'dict' objects. The public methods of *Dict are // thread safe. type Dict struct { Object table *dictTable // We use a recursive mutex for synchronization because the hash and // key comparison operations may re-enter DelItem/SetItem. mutex recursiveMutex // version is incremented whenever the Dict is modified. See: // https://www.python.org/dev/peps/pep-0509/ version int64 } // NewDict returns an empty Dict. func NewDict() *Dict { return &Dict{Object: Object{typ: DictType}, table: newDictTable(0)} } func newStringDict(items map[string]*Object) *Dict { if len(items) > maxDictSize/2 { panic(fmt.Sprintf("dictionary too big: %d", len(items))) } n := len(items) * 2 table := newDictTable(n) for key, value := range items { table.insertAbsentEntry(&dictEntry{hashString(key), NewStr(key).ToObject(), value}) } return &Dict{Object: Object{typ: DictType}, table: table} } func toDictUnsafe(o *Object) *Dict { return (*Dict)(o.toPointer()) } // loadTable atomically loads and returns d's underlying dictTable. func (d *Dict) loadTable() *dictTable { p := (*unsafe.Pointer)(unsafe.Pointer(&d.table)) return (*dictTable)(atomic.LoadPointer(p)) } // storeTable atomically updates d's underlying dictTable to the one given. func (d *Dict) storeTable(table *dictTable) { p := (*unsafe.Pointer)(unsafe.Pointer(&d.table)) atomic.StorePointer(p, unsafe.Pointer(table)) } // loadVersion atomically loads and returns d's version. func (d *Dict) loadVersion() int64 { // 64bit atomic ops need to be 8 byte aligned. This compile time check // verifies alignment by creating a negative constant for an unsigned type. // See sync/atomic docs for details. const blank = -(unsafe.Offsetof(d.version) % 8) return atomic.LoadInt64(&d.version) } // incVersion atomically increments d's version. func (d *Dict) incVersion() { // 64bit atomic ops need to be 8 byte aligned. This compile time check // verifies alignment by creating a negative constant for an unsigned type. // See sync/atomic docs for details. const blank = -(unsafe.Offsetof(d.version) % 8) atomic.AddInt64(&d.version, 1) } // DelItem removes the entry associated with key from d. It returns true if an // item was removed, or false if it did not exist in d. func (d *Dict) DelItem(f *Frame, key *Object) (bool, *BaseException) { originValue, raised := d.putItem(f, key, nil, true) if raised != nil { return false, raised } return originValue != nil, nil } // DelItemString removes the entry associated with key from d. It returns true // if an item was removed, or false if it did not exist in d. func (d *Dict) DelItemString(f *Frame, key string) (bool, *BaseException) { return d.DelItem(f, NewStr(key).ToObject()) } // GetItem looks up key in d, returning the associated value or nil if key is // not present in d. func (d *Dict) GetItem(f *Frame, key *Object) (*Object, *BaseException) { hash, raised := Hash(f, key) if raised != nil { return nil, raised } _, entry, raised := d.loadTable().lookupEntry(f, hash.Value(), key) if raised != nil { return nil, raised } if entry != nil && entry != deletedEntry { return entry.value, nil } return nil, nil } // GetItemString looks up key in d, returning the associated value or nil if // key is not present in d. func (d *Dict) GetItemString(f *Frame, key string) (*Object, *BaseException) { return d.GetItem(f, NewStr(key).ToObject()) } // Pop looks up key in d, returning and removing the associalted value if exist, // or nil if key is not present in d. func (d *Dict) Pop(f *Frame, key *Object) (*Object, *BaseException) { return d.putItem(f, key, nil, true) } // Keys returns a list containing all the keys in d. func (d *Dict) Keys(f *Frame) *List { d.mutex.Lock(f) keys := make([]*Object, d.Len()) i := 0 for _, entry := range d.table.entries { if entry != nil && entry != deletedEntry { keys[i] = entry.key i++ } } d.mutex.Unlock(f) return NewList(keys...) } // Len returns the number of entries in d. func (d *Dict) Len() int { return d.loadTable().loadUsed() } // putItem associates value with key in d, returning the old associated value if // the key was added, or nil if it was not already present in d. func (d *Dict) putItem(f *Frame, key, value *Object, overwrite bool) (*Object, *BaseException) { hash, raised := Hash(f, key) if raised != nil { return nil, raised } d.mutex.Lock(f) t := d.table v := d.version index, entry, raised := t.lookupEntry(f, hash.Value(), key) var originValue *Object if raised == nil { if v != d.version { // Dictionary was recursively modified. Blow up instead // of trying to recover. raised = f.RaiseType(RuntimeErrorType, "dictionary changed during write") } else { if value == nil { // Going to delete the entry. if entry != nil && entry != deletedEntry { d.table.storeEntry(index, deletedEntry) d.table.incUsed(-1) d.incVersion() } } else if overwrite || entry == nil { newEntry := &dictEntry{hash.Value(), key, value} if newTable, ok := t.writeEntry(f, index, newEntry); ok { if newTable != nil { d.storeTable(newTable) } d.incVersion() } else { raised = f.RaiseType(OverflowErrorType, errResultTooLarge) } } if entry != nil && entry != deletedEntry { originValue = entry.value } } } d.mutex.Unlock(f) return originValue, raised } // SetItem associates value with key in d. func (d *Dict) SetItem(f *Frame, key, value *Object) *BaseException { _, raised := d.putItem(f, key, value, true) return raised } // SetItemString associates value with key in d. func (d *Dict) SetItemString(f *Frame, key string, value *Object) *BaseException { return d.SetItem(f, NewStr(key).ToObject(), value) } // ToObject upcasts d to an Object. func (d *Dict) ToObject() *Object { return &d.Object } // Update copies the items from the mapping or sequence of 2-tuples o into d. func (d *Dict) Update(f *Frame, o *Object) (raised *BaseException) { var iter *Object if o.isInstance(DictType) { d2 := toDictUnsafe(o) d2.mutex.Lock(f) // Concurrent modifications to d2 will cause Update to raise // "dictionary changed during iteration". iter = newDictItemIterator(d2).ToObject() d2.mutex.Unlock(f) } else { iter, raised = Iter(f, o) } if raised != nil { return raised } return seqForEach(f, iter, func(item *Object) *BaseException { return seqApply(f, item, func(elems []*Object, _ bool) *BaseException { if numElems := len(elems); numElems != 2 { format := "dictionary update sequence element has length %d; 2 is required" return f.RaiseType(ValueErrorType, fmt.Sprintf(format, numElems)) } return d.SetItem(f, elems[0], elems[1]) }) }) } // dictsAreEqual returns true if d1 and d2 have the same keys and values, false // otherwise. If either d1 or d2 are concurrently modified then RuntimeError is // raised. func dictsAreEqual(f *Frame, d1, d2 *Dict) (bool, *BaseException) { if d1 == d2 { return true, nil } // Do not hold both locks at the same time to avoid deadlock. d1.mutex.Lock(f) iter := newDictEntryIterator(d1) g1 := newDictVersionGuard(d1) len1 := d1.Len() d1.mutex.Unlock(f) d2.mutex.Lock(f) g2 := newDictVersionGuard(d1) len2 := d2.Len() d2.mutex.Unlock(f) if len1 != len2 { return false, nil } result := true for entry := iter.next(); entry != nil && result; entry = iter.next() { if v, raised := d2.GetItem(f, entry.key); raised != nil { return false, raised } else if v == nil { result = false } else { eq, raised := Eq(f, entry.value, v) if raised != nil { return false, raised } result, raised = IsTrue(f, eq) if raised != nil { return false, raised } } } if !g1.check() || !g2.check() { return false, f.RaiseType(RuntimeErrorType, "dictionary changed during iteration") } return result, nil } func dictClear(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "clear", args, DictType); raised != nil { return nil, raised } d := toDictUnsafe(args[0]) d.mutex.Lock(f) d.table = newDictTable(0) d.incVersion() d.mutex.Unlock(f) return None, nil } func dictContains(f *Frame, seq, value *Object) (*Object, *BaseException) { item, raised := toDictUnsafe(seq).GetItem(f, value) if raised != nil { return nil, raised } return GetBool(item != nil).ToObject(), nil } func dictCopy(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "copy", args, DictType); raised != nil { return nil, raised } return DictType.Call(f, args, nil) } func dictDelItem(f *Frame, o, key *Object) *BaseException { deleted, raised := toDictUnsafe(o).DelItem(f, key) if raised != nil { return raised } if !deleted { return raiseKeyError(f, key) } return nil } func dictEq(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(DictType) { return NotImplemented, nil } eq, raised := dictsAreEqual(f, toDictUnsafe(v), toDictUnsafe(w)) if raised != nil { return nil, raised } return GetBool(eq).ToObject(), nil } func dictGet(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{DictType, ObjectType, ObjectType} argc := len(args) if argc == 2 { expectedTypes = expectedTypes[:2] } if raised := checkMethodArgs(f, "get", args, expectedTypes...); raised != nil { return nil, raised } item, raised := toDictUnsafe(args[0]).GetItem(f, args[1]) if raised == nil && item == nil { item = None if argc > 2 { item = args[2] } } return item, raised } func dictHasKey(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "has_key", args, DictType, ObjectType); raised != nil { return nil, raised } return dictContains(f, args[0], args[1]) } func dictItems(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "items", args, DictType); raised != nil { return nil, raised } d := toDictUnsafe(args[0]) d.mutex.Lock(f) iter := newDictItemIterator(d).ToObject() d.mutex.Unlock(f) return ListType.Call(f, Args{iter}, nil) } func dictIterItems(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "iteritems", args, DictType); raised != nil { return nil, raised } d := toDictUnsafe(args[0]) d.mutex.Lock(f) iter := newDictItemIterator(d).ToObject() d.mutex.Unlock(f) return iter, nil } func dictIterKeys(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "iterkeys", args, DictType); raised != nil { return nil, raised } return dictIter(f, args[0]) } func dictIterValues(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "itervalues", args, DictType); raised != nil { return nil, raised } d := toDictUnsafe(args[0]) d.mutex.Lock(f) iter := newDictValueIterator(d).ToObject() d.mutex.Unlock(f) return iter, nil } func dictKeys(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "keys", args, DictType); raised != nil { return nil, raised } return toDictUnsafe(args[0]).Keys(f).ToObject(), nil } func dictGetItem(f *Frame, o, key *Object) (*Object, *BaseException) { item, raised := toDictUnsafe(o).GetItem(f, key) if raised != nil { return nil, raised } if item == nil { return nil, raiseKeyError(f, key) } return item, nil } func dictInit(f *Frame, o *Object, args Args, kwargs KWArgs) (*Object, *BaseException) { var expectedTypes []*Type argc := len(args) if argc > 0 { expectedTypes = []*Type{ObjectType} } if raised := checkFunctionArgs(f, "__init__", args, expectedTypes...); raised != nil { return nil, raised } d := toDictUnsafe(o) if argc > 0 { if raised := d.Update(f, args[0]); raised != nil { return nil, raised } } for _, kwarg := range kwargs { if raised := d.SetItemString(f, kwarg.Name, kwarg.Value); raised != nil { return nil, raised } } return None, nil } func dictIter(f *Frame, o *Object) (*Object, *BaseException) { d := toDictUnsafe(o) d.mutex.Lock(f) iter := newDictKeyIterator(d).ToObject() d.mutex.Unlock(f) return iter, nil } func dictLen(f *Frame, o *Object) (*Object, *BaseException) { d := toDictUnsafe(o) ret := NewInt(d.Len()).ToObject() return ret, nil } func dictNE(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(DictType) { return NotImplemented, nil } eq, raised := dictsAreEqual(f, toDictUnsafe(v), toDictUnsafe(w)) if raised != nil { return nil, raised } return GetBool(!eq).ToObject(), nil } func dictNew(f *Frame, t *Type, _ Args, _ KWArgs) (*Object, *BaseException) { d := toDictUnsafe(newObject(t)) d.table = &dictTable{entries: make([]*dictEntry, minDictSize, minDictSize)} return d.ToObject(), nil } func dictPop(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{DictType, ObjectType, ObjectType} argc := len(args) if argc == 2 { expectedTypes = expectedTypes[:2] } if raised := checkMethodArgs(f, "pop", args, expectedTypes...); raised != nil { return nil, raised } key := args[1] d := toDictUnsafe(args[0]) item, raised := d.Pop(f, key) if raised == nil && item == nil { if argc > 2 { item = args[2] } else { raised = raiseKeyError(f, key) } } return item, raised } func dictPopItem(f *Frame, args Args, _ KWArgs) (item *Object, raised *BaseException) { if raised := checkMethodArgs(f, "popitem", args, DictType); raised != nil { return nil, raised } d := toDictUnsafe(args[0]) d.mutex.Lock(f) iter := newDictEntryIterator(d) entry := iter.next() if entry == nil { raised = f.RaiseType(KeyErrorType, "popitem(): dictionary is empty") } else { item = NewTuple(entry.key, entry.value).ToObject() d.table.storeEntry(int(iter.index-1), deletedEntry) d.table.incUsed(-1) d.incVersion() } d.mutex.Unlock(f) return item, raised } func dictRepr(f *Frame, o *Object) (*Object, *BaseException) { d := toDictUnsafe(o) if f.reprEnter(d.ToObject()) { return NewStr("{...}").ToObject(), nil } defer f.reprLeave(d.ToObject()) // Lock d so that we get a consistent view of it. Otherwise we may // return a state that d was never actually in. d.mutex.Lock(f) defer d.mutex.Unlock(f) var buf bytes.Buffer buf.WriteString("{") iter := newDictEntryIterator(d) i := 0 for entry := iter.next(); entry != nil; entry = iter.next() { if i > 0 { buf.WriteString(", ") } s, raised := Repr(f, entry.key) if raised != nil { return nil, raised } buf.WriteString(s.Value()) buf.WriteString(": ") if s, raised = Repr(f, entry.value); raised != nil { return nil, raised } buf.WriteString(s.Value()) i++ } buf.WriteString("}") return NewStr(buf.String()).ToObject(), nil } func dictSetDefault(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) if argc == 1 { return nil, f.RaiseType(TypeErrorType, "setdefault expected at least 1 arguments, got 0") } if argc > 3 { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("setdefault expected at most 2 arguments, got %v", argc-1)) } expectedTypes := []*Type{DictType, ObjectType, ObjectType} if argc == 2 { expectedTypes = expectedTypes[:2] } if raised := checkMethodArgs(f, "setdefault", args, expectedTypes...); raised != nil { return nil, raised } d := toDictUnsafe(args[0]) key := args[1] var value *Object if argc > 2 { value = args[2] } else { value = None } originValue, raised := d.putItem(f, key, value, false) if originValue != nil { return originValue, raised } return value, raised } func dictSetItem(f *Frame, o, key, value *Object) *BaseException { return toDictUnsafe(o).SetItem(f, key, value) } func dictUpdate(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{DictType, ObjectType} argc := len(args) if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkMethodArgs(f, "update", args, expectedTypes...); raised != nil { return nil, raised } d := toDictUnsafe(args[0]) if argc > 1 { if raised := d.Update(f, args[1]); raised != nil { return nil, raised } } for _, kwarg := range kwargs { if raised := d.SetItemString(f, kwarg.Name, kwarg.Value); raised != nil { return nil, raised } } return None, nil } func dictValues(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "values", args, DictType); raised != nil { return nil, raised } iter, raised := dictIterValues(f, args, nil) if raised != nil { return nil, raised } return ListType.Call(f, Args{iter}, nil) } func initDictType(dict map[string]*Object) { dict["clear"] = newBuiltinFunction("clear", dictClear).ToObject() dict["copy"] = newBuiltinFunction("copy", dictCopy).ToObject() dict["get"] = newBuiltinFunction("get", dictGet).ToObject() dict["has_key"] = newBuiltinFunction("has_key", dictHasKey).ToObject() dict["items"] = newBuiltinFunction("items", dictItems).ToObject() dict["iteritems"] = newBuiltinFunction("iteritems", dictIterItems).ToObject() dict["iterkeys"] = newBuiltinFunction("iterkeys", dictIterKeys).ToObject() dict["itervalues"] = newBuiltinFunction("itervalues", dictIterValues).ToObject() dict["keys"] = newBuiltinFunction("keys", dictKeys).ToObject() dict["pop"] = newBuiltinFunction("pop", dictPop).ToObject() dict["popitem"] = newBuiltinFunction("popitem", dictPopItem).ToObject() dict["setdefault"] = newBuiltinFunction("setdefault", dictSetDefault).ToObject() dict["update"] = newBuiltinFunction("update", dictUpdate).ToObject() dict["values"] = newBuiltinFunction("values", dictValues).ToObject() DictType.slots.Contains = &binaryOpSlot{dictContains} DictType.slots.DelItem = &delItemSlot{dictDelItem} DictType.slots.Eq = &binaryOpSlot{dictEq} DictType.slots.GetItem = &binaryOpSlot{dictGetItem} DictType.slots.Hash = &unaryOpSlot{hashNotImplemented} DictType.slots.Init = &initSlot{dictInit} DictType.slots.Iter = &unaryOpSlot{dictIter} DictType.slots.Len = &unaryOpSlot{dictLen} DictType.slots.NE = &binaryOpSlot{dictNE} DictType.slots.New = &newSlot{dictNew} DictType.slots.Repr = &unaryOpSlot{dictRepr} DictType.slots.SetItem = &setItemSlot{dictSetItem} } type dictItemIterator struct { Object iter dictEntryIterator guard dictVersionGuard } // newDictItemIterator creates a dictItemIterator object for d. It assumes that // d.mutex is held by the caller. func newDictItemIterator(d *Dict) *dictItemIterator { return &dictItemIterator{ Object: Object{typ: dictItemIteratorType}, iter: newDictEntryIterator(d), guard: newDictVersionGuard(d), } } func toDictItemIteratorUnsafe(o *Object) *dictItemIterator { return (*dictItemIterator)(o.toPointer()) } func (iter *dictItemIterator) ToObject() *Object { return &iter.Object } func dictItemIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func dictItemIteratorNext(f *Frame, o *Object) (ret *Object, raised *BaseException) { iter := toDictItemIteratorUnsafe(o) entry, raised := dictIteratorNext(f, &iter.iter, &iter.guard) if raised != nil { return nil, raised } return NewTuple2(entry.key, entry.value).ToObject(), nil } func initDictItemIteratorType(map[string]*Object) { dictItemIteratorType.flags &^= typeFlagBasetype | typeFlagInstantiable dictItemIteratorType.slots.Iter = &unaryOpSlot{dictItemIteratorIter} dictItemIteratorType.slots.Next = &unaryOpSlot{dictItemIteratorNext} } type dictKeyIterator struct { Object iter dictEntryIterator guard dictVersionGuard } // newDictKeyIterator creates a dictKeyIterator object for d. It assumes that // d.mutex is held by the caller. func newDictKeyIterator(d *Dict) *dictKeyIterator { return &dictKeyIterator{ Object: Object{typ: dictKeyIteratorType}, iter: newDictEntryIterator(d), guard: newDictVersionGuard(d), } } func toDictKeyIteratorUnsafe(o *Object) *dictKeyIterator { return (*dictKeyIterator)(o.toPointer()) } func (iter *dictKeyIterator) ToObject() *Object { return &iter.Object } func dictKeyIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func dictKeyIteratorNext(f *Frame, o *Object) (*Object, *BaseException) { iter := toDictKeyIteratorUnsafe(o) entry, raised := dictIteratorNext(f, &iter.iter, &iter.guard) if raised != nil { return nil, raised } return entry.key, nil } func initDictKeyIteratorType(map[string]*Object) { dictKeyIteratorType.flags &^= typeFlagBasetype | typeFlagInstantiable dictKeyIteratorType.slots.Iter = &unaryOpSlot{dictKeyIteratorIter} dictKeyIteratorType.slots.Next = &unaryOpSlot{dictKeyIteratorNext} } type dictValueIterator struct { Object iter dictEntryIterator guard dictVersionGuard } // newDictValueIterator creates a dictValueIterator object for d. It assumes // that d.mutex is held by the caller. func newDictValueIterator(d *Dict) *dictValueIterator { return &dictValueIterator{ Object: Object{typ: dictValueIteratorType}, iter: newDictEntryIterator(d), guard: newDictVersionGuard(d), } } func toDictValueIteratorUnsafe(o *Object) *dictValueIterator { return (*dictValueIterator)(o.toPointer()) } func (iter *dictValueIterator) ToObject() *Object { return &iter.Object } func dictValueIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func dictValueIteratorNext(f *Frame, o *Object) (*Object, *BaseException) { iter := toDictValueIteratorUnsafe(o) entry, raised := dictIteratorNext(f, &iter.iter, &iter.guard) if raised != nil { return nil, raised } return entry.value, nil } func initDictValueIteratorType(map[string]*Object) { dictValueIteratorType.flags &^= typeFlagBasetype | typeFlagInstantiable dictValueIteratorType.slots.Iter = &unaryOpSlot{dictValueIteratorIter} dictValueIteratorType.slots.Next = &unaryOpSlot{dictValueIteratorNext} } func raiseKeyError(f *Frame, key *Object) *BaseException { s, raised := ToStr(f, key) if raised == nil { raised = f.RaiseType(KeyErrorType, s.Value()) } return raised } func dictNextIndex(i, perturb uint) (uint, uint) { return (i << 2) + i + perturb + 1, perturb >> 5 } func dictIteratorNext(f *Frame, iter *dictEntryIterator, guard *dictVersionGuard) (*dictEntry, *BaseException) { // NOTE: The behavior here diverges from CPython where an iterator that // is exhausted will always return StopIteration regardless whether the // underlying dict is subsequently modified. In Grumpy, an iterator for // a dict that has been modified will always raise RuntimeError even if // the iterator was exhausted before the modification. entry := iter.next() if !guard.check() { return nil, f.RaiseType(RuntimeErrorType, "dictionary changed during iteration") } if entry == nil { return nil, f.Raise(StopIterationType.ToObject(), nil, nil) } return entry, nil } ================================================ FILE: runtime/dict_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "reflect" "regexp" "runtime" "sync" "testing" "time" ) // hashFoo is the hash of the string 'foo'. We use this to validate some corner // cases around hash collision below. // NOTE: Inline func helps support 32bit systems. var hashFoo = NewInt(func(i int64) int { return int(i) }(-4177197833195190597)).ToObject() func TestNewStringDict(t *testing.T) { cases := []struct { m map[string]*Object want *Dict }{ {nil, NewDict()}, {map[string]*Object{"baz": NewFloat(3.14).ToObject()}, newTestDict("baz", 3.14)}, {map[string]*Object{"foo": NewInt(2).ToObject(), "bar": NewInt(4).ToObject()}, newTestDict("bar", 4, "foo", 2)}, } for _, cas := range cases { fun := newBuiltinFunction("newStringDict", func(*Frame, Args, KWArgs) (*Object, *BaseException) { return newStringDict(cas.m).ToObject(), nil }).ToObject() if err := runInvokeTestCase(fun, &invokeTestCase{want: cas.want.ToObject()}); err != "" { t.Error(err) } } } func TestDictClear(t *testing.T) { clear := mustNotRaise(GetAttr(NewRootFrame(), DictType.ToObject(), NewStr("clear"), nil)) fun := newBuiltinFunction("TestDictClear", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if _, raised := clear.Call(f, args, nil); raised != nil { return nil, raised } return args[0], nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(NewDict()), want: NewDict().ToObject()}, {args: wrapArgs(newStringDict(map[string]*Object{"foo": NewInt(1).ToObject()})), want: NewDict().ToObject()}, {args: wrapArgs(newTestDict(2, None, "baz", 3.14)), want: NewDict().ToObject()}, {args: wrapArgs(NewDict(), NewList()), wantExc: mustCreateException(TypeErrorType, "'clear' of 'dict' requires 1 arguments")}, {args: wrapArgs(NewDict(), None), wantExc: mustCreateException(TypeErrorType, "'clear' of 'dict' requires 1 arguments")}, {args: wrapArgs(None), wantExc: mustCreateException(TypeErrorType, "unbound method clear() must be called with dict instance as first argument (got NoneType instance instead)")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestDictContains(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewDict(), "foo"), want: False.ToObject()}, {args: wrapArgs(newTestDict("foo", 1, "bar", 2), "foo"), want: True.ToObject()}, {args: wrapArgs(newTestDict(3, "foo", "bar", 42), 42), want: False.ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(DictType, "__contains__", &cas); err != "" { t.Error(err) } } } func TestDictDelItem(t *testing.T) { fun := newBuiltinFunction("TestDictDelItem", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "TestDictDelItem", args, DictType, ObjectType); raised != nil { return nil, raised } if raised := DelItem(f, args[0], args[1]); raised != nil { return nil, raised } return args[0], nil }).ToObject() testDict := newTestDict("a", 1, "b", 2, "c", 3) cases := []invokeTestCase{ {args: wrapArgs(newTestDict("foo", 1), "foo"), want: NewDict().ToObject()}, {args: wrapArgs(NewDict(), 10), wantExc: mustCreateException(KeyErrorType, "10")}, {args: wrapArgs(testDict, "a"), want: newTestDict("b", 2, "c", 3).ToObject()}, {args: wrapArgs(testDict, "c"), want: newTestDict("b", 2).ToObject()}, {args: wrapArgs(testDict, "a"), wantExc: mustCreateException(KeyErrorType, "a")}, {args: wrapArgs(NewDict(), NewList()), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestDictDelItemString(t *testing.T) { fun := newBuiltinFunction("TestDictDelItemString", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "TestDictDelItemString", args, DictType, StrType); raised != nil { return nil, raised } deleted, raised := toDictUnsafe(args[0]).DelItemString(f, toStrUnsafe(args[1]).Value()) if raised != nil { return nil, raised } return newTestTuple(deleted, args[0]).ToObject(), nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(newTestDict("foo", 1), "foo"), want: newTestTuple(true, NewDict()).ToObject()}, {args: wrapArgs(NewDict(), "qux"), want: newTestTuple(false, NewDict()).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestDictEqNE(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, v, w *Object) (*Object, *BaseException) { eq, raised := Eq(f, v, w) if raised != nil { return nil, raised } ne, raised := NE(f, v, w) if raised != nil { return nil, raised } valid := GetBool(eq == True.ToObject() && ne == False.ToObject() || eq == False.ToObject() && ne == True.ToObject()).ToObject() if raised := Assert(f, valid, NewStr("invalid values for __eq__ or __ne__").ToObject()); raised != nil { return nil, raised } return eq, nil }) f := NewRootFrame() large1, large2 := NewDict(), NewDict() largeSize := 100 for i := 0; i < largeSize; i++ { s, raised := ToStr(f, NewInt(i).ToObject()) if raised != nil { t.Fatal(raised) } large1.SetItem(f, NewInt(i).ToObject(), s.ToObject()) s, raised = ToStr(f, NewInt(largeSize-i-1).ToObject()) if raised != nil { t.Fatal(raised) } large2.SetItem(f, NewInt(largeSize-i-1).ToObject(), s.ToObject()) } o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(NewDict(), NewDict()), want: True.ToObject()}, {args: wrapArgs(NewDict(), newTestDict("foo", true)), want: False.ToObject()}, {args: wrapArgs(newTestDict("foo", "foo"), newTestDict("foo", "foo")), want: True.ToObject()}, {args: wrapArgs(newTestDict("foo", true), newTestDict("bar", true)), want: False.ToObject()}, {args: wrapArgs(newTestDict("foo", true), newTestDict("foo", newObject(ObjectType))), want: False.ToObject()}, {args: wrapArgs(newTestDict("foo", true, "bar", false), newTestDict("bar", true)), want: False.ToObject()}, {args: wrapArgs(newTestDict("foo", o, "bar", o), newTestDict("foo", o, "bar", o)), want: True.ToObject()}, {args: wrapArgs(newTestDict(2, None, "foo", o), newTestDict("foo", o, 2, None)), want: True.ToObject()}, {args: wrapArgs(large1, large2), want: True.ToObject()}, {args: wrapArgs(NewDict(), 123), want: False.ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestDictGet(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewDict(), "foo"), want: None}, {args: wrapArgs(newTestDict("foo", 1, "bar", 2), "foo"), want: NewInt(1).ToObject()}, {args: wrapArgs(newTestDict(3, "foo", "bar", 42), 42, "nope"), want: NewStr("nope").ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(DictType, "get", &cas); err != "" { t.Error(err) } } } func TestDictGetItem(t *testing.T) { getItem := newBuiltinFunction("TestDictGetItem", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestDictGetItem", args, DictType, ObjectType); raised != nil { return nil, raised } result, raised := toDictUnsafe(args[0]).GetItem(f, args[1]) if raised == nil && result == nil { result = None } return result, raised }).ToObject() f := NewRootFrame() h, raised := Hash(f, NewStr("foo").ToObject()) if raised != nil { t.Fatal(raised) } if b, raised := IsTrue(f, mustNotRaise(NE(f, h.ToObject(), hashFoo))); raised != nil { t.Fatal(raised) } else if b { t.Fatalf("hash('foo') = %v, want %v", h, hashFoo) } deletedItemDict := newTestDict(hashFoo, true, "foo", true) deletedItemDict.DelItem(f, hashFoo) cases := []invokeTestCase{ {args: wrapArgs(NewDict(), "foo"), want: None}, {args: wrapArgs(newStringDict(map[string]*Object{"foo": True.ToObject()}), "foo"), want: True.ToObject()}, {args: wrapArgs(newTestDict(2, "bar", "baz", 3.14), 2), want: NewStr("bar").ToObject()}, {args: wrapArgs(newTestDict(2, "bar", "baz", 3.14), 3), want: None}, {args: wrapArgs(deletedItemDict, hashFoo), want: None}, {args: wrapArgs(NewDict(), NewList()), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, } for _, cas := range cases { if err := runInvokeTestCase(getItem, &cas); err != "" { t.Error(err) } } } // BenchmarkDictGetItem is to keep an eye on the speed of contended dict access // in a fast read loop. func BenchmarkDictGetItem(b *testing.B) { d := newTestDict( "foo", 1, "bar", 2, None, 3, 4, 5) k := NewInt(4).ToObject() b.ResetTimer() b.RunParallel(func(pb *testing.PB) { f := NewRootFrame() var ret *Object var raised *BaseException for pb.Next() { ret, raised = d.GetItem(f, k) } runtime.KeepAlive(ret) runtime.KeepAlive(raised) }) } func BenchmarkDictIterItems(b *testing.B) { bench := func(d *Dict) func(*testing.B) { return func(b *testing.B) { f := NewRootFrame() args := f.MakeArgs(1) args[0] = d.ToObject() b.ResetTimer() var ret *Object var raised *BaseException for i := 0; i < b.N; i++ { iter, _ := dictIterItems(f, args, nil) for { ret, raised = Next(f, iter) if raised != nil { if !raised.isInstance(StopIterationType) { b.Fatalf("iteration failed with: %v", raised) } f.RestoreExc(nil, nil) break } } } runtime.KeepAlive(ret) runtime.KeepAlive(raised) } } b.Run("0-elements", bench(newTestDict())) b.Run("1-elements", bench(newTestDict(1, 2))) b.Run("2-elements", bench(newTestDict(1, 2, 3, 4))) b.Run("3-elements", bench(newTestDict(1, 2, 3, 4, 5, 6))) b.Run("4-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8))) b.Run("5-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))) b.Run("6-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12))) b.Run("7-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14))) b.Run("8-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16))) } func BenchmarkDictIterKeys(b *testing.B) { bench := func(d *Dict) func(*testing.B) { return func(b *testing.B) { f := NewRootFrame() args := f.MakeArgs(1) args[0] = d.ToObject() b.ResetTimer() var ret *Object var raised *BaseException for i := 0; i < b.N; i++ { iter, _ := dictIterKeys(f, args, nil) for { ret, raised = Next(f, iter) if raised != nil { if !raised.isInstance(StopIterationType) { b.Fatalf("iteration failed with: %v", raised) } f.RestoreExc(nil, nil) break } } } runtime.KeepAlive(ret) runtime.KeepAlive(raised) } } b.Run("0-elements", bench(newTestDict())) b.Run("1-elements", bench(newTestDict(1, 2))) b.Run("2-elements", bench(newTestDict(1, 2, 3, 4))) b.Run("3-elements", bench(newTestDict(1, 2, 3, 4, 5, 6))) b.Run("4-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8))) b.Run("5-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))) b.Run("6-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12))) b.Run("7-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14))) b.Run("8-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16))) } func BenchmarkDictIterValues(b *testing.B) { bench := func(d *Dict) func(*testing.B) { return func(b *testing.B) { f := NewRootFrame() args := f.MakeArgs(1) args[0] = d.ToObject() b.ResetTimer() var ret *Object var raised *BaseException for i := 0; i < b.N; i++ { iter, _ := dictIterValues(f, args, nil) for { ret, raised = Next(f, iter) if raised != nil { if !raised.isInstance(StopIterationType) { b.Fatalf("iteration failed with: %v", raised) } f.RestoreExc(nil, nil) break } } } runtime.KeepAlive(ret) runtime.KeepAlive(raised) } } b.Run("0-elements", bench(newTestDict())) b.Run("1-elements", bench(newTestDict(1, 2))) b.Run("2-elements", bench(newTestDict(1, 2, 3, 4))) b.Run("3-elements", bench(newTestDict(1, 2, 3, 4, 5, 6))) b.Run("4-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8))) b.Run("5-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))) b.Run("6-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12))) b.Run("7-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14))) b.Run("8-elements", bench(newTestDict(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16))) } func TestDictGetItemString(t *testing.T) { getItemString := newBuiltinFunction("TestDictGetItemString", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestDictGetItem", args, DictType, StrType); raised != nil { return nil, raised } result, raised := toDictUnsafe(args[0]).GetItemString(f, toStrUnsafe(args[1]).Value()) if raised == nil && result == nil { result = None } return result, raised }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(NewDict(), "foo"), want: None}, {args: wrapArgs(newTestDict("foo", true), "foo"), want: True.ToObject()}, {args: wrapArgs(newTestDict(2, "bar", "baz", 3.14), "baz"), want: NewFloat(3.14).ToObject()}, {args: wrapArgs(newTestDict(2, "bar", "baz", 3.14), "qux"), want: None}, } for _, cas := range cases { if err := runInvokeTestCase(getItemString, &cas); err != "" { t.Error(err) } } } func TestDictHasKey(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewDict(), "foo"), want: False.ToObject()}, {args: wrapArgs(newTestDict("foo", 1, "bar", 2), "foo"), want: True.ToObject()}, {args: wrapArgs(newTestDict(3, "foo", "bar", 42), 42), want: False.ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(DictType, "has_key", &cas); err != "" { t.Error(err) } } } func TestDictItemIteratorIter(t *testing.T) { iter := &newDictItemIterator(NewDict()).Object cas := &invokeTestCase{args: wrapArgs(iter), want: iter} if err := runInvokeMethodTestCase(dictItemIteratorType, "__iter__", cas); err != "" { t.Error(err) } } func TestDictItemIterModified(t *testing.T) { f := NewRootFrame() iterItems := mustNotRaise(GetAttr(f, DictType.ToObject(), NewStr("iteritems"), nil)) d := NewDict() iter := mustNotRaise(iterItems.Call(f, wrapArgs(d), nil)) if raised := d.SetItemString(f, "foo", None); raised != nil { t.Fatal(raised) } cas := invokeTestCase{ args: wrapArgs(iter), wantExc: mustCreateException(RuntimeErrorType, "dictionary changed during iteration"), } if err := runInvokeMethodTestCase(dictItemIteratorType, "next", &cas); err != "" { t.Error(err) } } func TestDictIter(t *testing.T) { iter := newBuiltinFunction("TestDictIter", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestDictIter", args, DictType); raised != nil { return nil, raised } iter, raised := Iter(f, args[0]) if raised != nil { return nil, raised } return TupleType.Call(f, []*Object{iter}, nil) }).ToObject() f := NewRootFrame() deletedItemDict := newTestDict(hashFoo, None, "foo", None) deletedItemDict.DelItem(f, hashFoo) cases := []invokeTestCase{ {args: wrapArgs(NewDict()), want: NewTuple().ToObject()}, {args: wrapArgs(newStringDict(map[string]*Object{"foo": NewInt(1).ToObject(), "bar": NewInt(2).ToObject()})), want: newTestTuple("foo", "bar").ToObject()}, {args: wrapArgs(newTestDict(123, True, "foo", False)), want: newTestTuple(123, "foo").ToObject()}, {args: wrapArgs(deletedItemDict), want: newTestTuple("foo").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(iter, &cas); err != "" { t.Error(err) } } } func TestDictIterKeys(t *testing.T) { iterkeys := mustNotRaise(GetAttr(NewRootFrame(), DictType.ToObject(), NewStr("iterkeys"), nil)) fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Object, *BaseException) { iter, raised := iterkeys.Call(f, args, nil) if raised != nil { return nil, raised } return TupleType.Call(f, Args{iter}, nil) }) cases := []invokeTestCase{ {args: wrapArgs(NewDict()), want: NewTuple().ToObject()}, {args: wrapArgs(newTestDict("foo", 1, "bar", 2)), want: newTestTuple("foo", "bar").ToObject()}, {args: wrapArgs(NewDict(), "bad"), wantExc: mustCreateException(TypeErrorType, "'iterkeys' of 'dict' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestDictIterValues(t *testing.T) { itervalues := mustNotRaise(GetAttr(NewRootFrame(), DictType.ToObject(), NewStr("itervalues"), nil)) fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Object, *BaseException) { iter, raised := itervalues.Call(f, args, nil) if raised != nil { return nil, raised } return TupleType.Call(f, Args{iter}, nil) }) cases := []invokeTestCase{ {args: wrapArgs(NewDict()), want: NewTuple().ToObject()}, {args: wrapArgs(newTestDict("foo", 1, "bar", 2)), want: newTestTuple(1, 2).ToObject()}, {args: wrapArgs(NewDict(), "bad"), wantExc: mustCreateException(TypeErrorType, "'itervalues' of 'dict' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } // Tests dict.items and dict.iteritems. func TestDictItems(t *testing.T) { f := NewRootFrame() iterItems := mustNotRaise(GetAttr(f, DictType.ToObject(), NewStr("iteritems"), nil)) items := newBuiltinFunction("TestDictIterItems", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestDictIterItems", args, DictType); raised != nil { return nil, raised } iter, raised := iterItems.Call(f, []*Object{args[0]}, nil) if raised != nil { return nil, raised } return ListType.Call(f, []*Object{iter}, nil) }).ToObject() deletedItemDict := newTestDict(hashFoo, None, "foo", None) deletedItemDict.DelItem(f, hashFoo) cases := []invokeTestCase{ {args: wrapArgs(NewDict()), want: NewList().ToObject()}, {args: wrapArgs(newStringDict(map[string]*Object{"foo": NewInt(1).ToObject(), "bar": NewInt(2).ToObject()})), want: newTestList(newTestTuple("foo", 1), newTestTuple("bar", 2)).ToObject()}, {args: wrapArgs(newTestDict(123, True, "foo", False)), want: newTestList(newTestTuple(123, true), newTestTuple("foo", false)).ToObject()}, {args: wrapArgs(deletedItemDict), want: newTestList(newTestTuple("foo", None)).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(items, &cas); err != "" { t.Error(err) } if err := runInvokeMethodTestCase(DictType, "items", &cas); err != "" { t.Error(err) } } } func TestDictKeyIteratorIter(t *testing.T) { iter := &newDictKeyIterator(NewDict()).Object cas := &invokeTestCase{args: wrapArgs(iter), want: iter} if err := runInvokeMethodTestCase(dictKeyIteratorType, "__iter__", cas); err != "" { t.Error(err) } } func TestDictKeyIterModified(t *testing.T) { f := NewRootFrame() d := NewDict() iter := mustNotRaise(Iter(f, d.ToObject())) if raised := d.SetItemString(f, "foo", None); raised != nil { t.Fatal(raised) } cas := invokeTestCase{ args: wrapArgs(iter), wantExc: mustCreateException(RuntimeErrorType, "dictionary changed during iteration"), } if err := runInvokeMethodTestCase(dictKeyIteratorType, "next", &cas); err != "" { t.Error(err) } } func TestDictKeys(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewDict()), want: NewList().ToObject()}, {args: wrapArgs(newTestDict("foo", None, 42, None)), want: newTestList(42, "foo").ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(DictType, "keys", &cas); err != "" { t.Error(err) } } } func TestDictPop(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestDict("foo", 42), "foo"), want: NewInt(42).ToObject()}, {args: wrapArgs(NewDict(), "foo", 42), want: NewInt(42).ToObject()}, {args: wrapArgs(NewDict(), "foo"), wantExc: mustCreateException(KeyErrorType, "foo")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(DictType, "pop", &cas); err != "" { t.Error(err) } } } func TestDictPopItem(t *testing.T) { popItem := mustNotRaise(GetAttr(NewRootFrame(), DictType.ToObject(), NewStr("popitem"), nil)) fun := wrapFuncForTest(func(f *Frame, d *Dict) (*Object, *BaseException) { result := NewDict() item, raised := popItem.Call(f, wrapArgs(d), nil) for ; raised == nil; item, raised = popItem.Call(f, wrapArgs(d), nil) { t := toTupleUnsafe(item) result.SetItem(f, t.GetItem(0), t.GetItem(1)) } if raised != nil { if !raised.isInstance(KeyErrorType) { return nil, raised } f.RestoreExc(nil, nil) } if raised = Assert(f, GetBool(d.Len() == 0).ToObject(), nil); raised != nil { return nil, raised } return result.ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(newTestDict("foo", 42)), want: newTestDict("foo", 42).ToObject()}, {args: wrapArgs(newTestDict("foo", 42, 123, "bar")), want: newTestDict("foo", 42, 123, "bar").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestDictNewInit(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(), want: NewDict().ToObject()}, {args: wrapArgs(newTestDict("foo", 42)), want: newTestDict("foo", 42).ToObject()}, {args: wrapArgs(), kwargs: wrapKWArgs("foo", 42), want: newTestDict("foo", 42).ToObject()}, {args: wrapArgs(newTestDict("foo", 42)), kwargs: wrapKWArgs("foo", "bar"), want: newTestDict("foo", "bar").ToObject()}, {args: wrapArgs(newTestList(newTestTuple("baz", 42))), kwargs: wrapKWArgs("foo", "bar"), want: newTestDict("baz", 42, "foo", "bar").ToObject()}, {args: wrapArgs(True), wantExc: mustCreateException(TypeErrorType, "'bool' object is not iterable")}, {args: wrapArgs(NewList(), "foo"), wantExc: mustCreateException(TypeErrorType, "'__init__' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(DictType.ToObject(), &cas); err != "" { t.Error(err) } } } func TestDictNewRaises(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'__new__' requires 1 arguments")}, {args: wrapArgs(123), wantExc: mustCreateException(TypeErrorType, `'__new__' requires a 'type' object but received a "int"`)}, {args: wrapArgs(NoneType), wantExc: mustCreateException(TypeErrorType, "dict.__new__(NoneType): NoneType is not a subtype of dict")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(DictType, "__new__", &cas); err != "" { t.Error(err) } } } func TestDictSetDefault(t *testing.T) { setDefaultMethod := mustNotRaise(GetAttr(NewRootFrame(), DictType.ToObject(), NewStr("setdefault"), nil)) setDefault := newBuiltinFunction("TestDictSetDefault", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { i, raised := setDefaultMethod.Call(f, args, kwargs) if raised != nil { return nil, raised } return NewTuple(i, args[0]).ToObject(), nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(NewDict(), "foo"), want: newTestTuple(None, newTestDict("foo", None)).ToObject()}, {args: wrapArgs(NewDict(), "foo", 42), want: newTestTuple(42, newTestDict("foo", 42)).ToObject()}, {args: wrapArgs(newTestDict("foo", 42), "foo"), want: newTestTuple(42, newTestDict("foo", 42)).ToObject()}, {args: wrapArgs(newTestDict("foo", 42), "foo", 43), want: newTestTuple(42, newTestDict("foo", 42)).ToObject()}, {args: wrapArgs(NewDict()), wantExc: mustCreateException(TypeErrorType, "setdefault expected at least 1 arguments, got 0")}, {args: wrapArgs(NewDict(), "foo", "bar", "baz"), wantExc: mustCreateException(TypeErrorType, "setdefault expected at most 2 arguments, got 3")}, } for _, cas := range cases { if err := runInvokeTestCase(setDefault, &cas); err != "" { t.Error(err) } } } func TestDictSetItem(t *testing.T) { setItem := newBuiltinFunction("TestDictSetItem", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestDictSetItem", args, DictType, ObjectType, ObjectType); raised != nil { return nil, raised } d := toDictUnsafe(args[0]) if raised := d.SetItem(f, args[1], args[2]); raised != nil { return nil, raised } return d.ToObject(), nil }).ToObject() f := NewRootFrame() o := newObject(ObjectType) deletedItemDict := newStringDict(map[string]*Object{"foo": None}) if _, raised := deletedItemDict.DelItemString(f, "foo"); raised != nil { t.Fatal(raised) } modifiedDict := newTestDict(0, None) modifiedType := newTestClass("Foo", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { for i := 1000; i < 1100; i++ { if raised := modifiedDict.SetItem(f, NewInt(i).ToObject(), None); raised != nil { return nil, raised } } return False.ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(NewDict(), "foo", o), want: newStringDict(map[string]*Object{"foo": o}).ToObject()}, {args: wrapArgs(newStringDict(map[string]*Object{"foo": NewInt(1).ToObject()}), "foo", 2), want: newStringDict(map[string]*Object{"foo": NewInt(2).ToObject()}).ToObject()}, {args: wrapArgs(newTestDict(2, None, "baz", 3.14), 2, o), want: newTestDict(2, o, "baz", 3.14).ToObject()}, {args: wrapArgs(deletedItemDict, "foo", o), want: newStringDict(map[string]*Object{"foo": o}).ToObject()}, {args: wrapArgs(NewDict(), NewList(), None), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, {args: wrapArgs(modifiedDict, newObject(modifiedType), None), wantExc: mustCreateException(RuntimeErrorType, "dictionary changed during write")}, } for _, cas := range cases { if err := runInvokeTestCase(setItem, &cas); err != "" { t.Error(err) } } } func TestDictSetItemString(t *testing.T) { setItemString := newBuiltinFunction("TestDictSetItemString", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestDictSetItemString", args, DictType, StrType, ObjectType); raised != nil { return nil, raised } d := toDictUnsafe(args[0]) if raised := d.SetItemString(f, toStrUnsafe(args[1]).Value(), args[2]); raised != nil { return nil, raised } return d.ToObject(), nil }).ToObject() o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(NewDict(), "foo", o), want: newStringDict(map[string]*Object{"foo": o}).ToObject()}, {args: wrapArgs(newStringDict(map[string]*Object{"foo": NewInt(1).ToObject()}), "foo", 2), want: newStringDict(map[string]*Object{"foo": NewInt(2).ToObject()}).ToObject()}, {args: wrapArgs(newTestDict(2, None, "baz", 3.14), "baz", o), want: newTestDict(2, None, "baz", o).ToObject()}, {args: wrapArgs(newTestDict(hashFoo, o, "foo", None), "foo", 3.14), want: newTestDict(hashFoo, o, "foo", 3.14).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(setItemString, &cas); err != "" { t.Error(err) } } } func TestDictStrRepr(t *testing.T) { recursiveDict := NewDict() if raised := recursiveDict.SetItemString(NewRootFrame(), "key", recursiveDict.ToObject()); raised != nil { t.Fatal(raised) } cases := []struct { o *Object wantPatterns []string }{ {NewDict().ToObject(), []string{"^{}$"}}, {newStringDict(map[string]*Object{"foo": NewStr("foo value").ToObject()}).ToObject(), []string{`^\{'foo': 'foo value'\}$`}}, {newStringDict(map[string]*Object{"foo": NewStr("foo value").ToObject(), "bar": NewStr("bar value").ToObject()}).ToObject(), []string{`^{.*, .*}$`, `'foo': 'foo value'`, `'bar': 'bar value'`}}, {recursiveDict.ToObject(), []string{`^{'key': {\.\.\.}}$`}}, } for _, cas := range cases { fun := wrapFuncForTest(func(f *Frame) *BaseException { for _, pattern := range cas.wantPatterns { re := regexp.MustCompile(pattern) s, raised := ToStr(f, cas.o) if raised != nil { return raised } if !re.MatchString(s.Value()) { t.Errorf("str(%v) = %v, want %q", cas.o, s, re) } s, raised = Repr(f, cas.o) if raised != nil { return raised } if !re.MatchString(s.Value()) { t.Errorf("repr(%v) = %v, want %q", cas.o, s, re) } } return nil }) if err := runInvokeTestCase(fun, &invokeTestCase{want: None}); err != "" { t.Error(err) } } } func TestDictUpdate(t *testing.T) { updateMethod := mustNotRaise(GetAttr(NewRootFrame(), DictType.ToObject(), NewStr("update"), nil)) update := newBuiltinFunction("TestDictUpdate", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionVarArgs(f, "TestDictUpdate", args, DictType); raised != nil { return nil, raised } if _, raised := updateMethod.Call(f, args, kwargs); raised != nil { return nil, raised } return args[0], nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(newTestDict(42, "foo")), want: newTestDict(42, "foo").ToObject()}, {args: wrapArgs(NewDict(), NewDict()), want: NewDict().ToObject()}, {args: wrapArgs(NewDict(), newTestDict("foo", 42, "bar", 43)), want: newTestDict("foo", 42, "bar", 43).ToObject()}, {args: wrapArgs(newTestDict(123, None), newTestDict(124, True)), want: newTestDict(123, None, 124, True).ToObject()}, {args: wrapArgs(newTestDict("foo", 3.14), newTestDict("foo", "bar")), want: newTestDict("foo", "bar").ToObject()}, {args: wrapArgs(NewDict(), NewTuple()), want: NewDict().ToObject()}, {args: wrapArgs(NewDict(), newTestList(newTestTuple("foo", 42), newTestTuple("bar", 43))), want: newTestDict("foo", 42, "bar", 43).ToObject()}, {args: wrapArgs(newTestDict(123, None), newTestTuple(newTestTuple(124, True))), want: newTestDict(123, None, 124, True).ToObject()}, {args: wrapArgs(newTestDict("foo", 3.14), newTestList(newTestList("foo", "bar"))), want: newTestDict("foo", "bar").ToObject()}, {args: wrapArgs(NewDict(), None), wantExc: mustCreateException(TypeErrorType, "'NoneType' object is not iterable")}, {args: wrapArgs(NewDict(), newTestTuple(newTestList(None, 42, "foo"))), wantExc: mustCreateException(ValueErrorType, "dictionary update sequence element has length 3; 2 is required")}, {args: wrapArgs(NewDict()), want: NewDict().ToObject()}, {args: wrapArgs(NewDict()), kwargs: wrapKWArgs("foo", "bar"), want: newTestDict("foo", "bar").ToObject()}, {args: wrapArgs(newTestDict("foo", 1, "bar", 3.14), newTestDict("foo", 2)), kwargs: wrapKWArgs("foo", 3), want: newTestDict("foo", 3, "bar", 3.14).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(update, &cas); err != "" { t.Error(err) } } } func TestDictValues(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewDict()), want: NewList().ToObject()}, {args: wrapArgs(newTestDict("foo", 1, "bar", 2)), want: newTestList(1, 2).ToObject()}, {args: wrapArgs(NewDict(), "bad"), wantExc: mustCreateException(TypeErrorType, "'values' of 'dict' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(DictType, "values", &cas); err != "" { t.Error(err) } } } func TestParallelDictUpdates(t *testing.T) { keys := []*Object{ NewStr("abc").ToObject(), NewStr("def").ToObject(), NewStr("ghi").ToObject(), NewStr("jkl").ToObject(), NewStr("mno").ToObject(), NewStr("pqr").ToObject(), NewStr("stu").ToObject(), NewStr("vwx").ToObject(), NewStr("yz0").ToObject(), NewStr("123").ToObject(), NewStr("456").ToObject(), NewStr("789").ToObject(), NewStr("ABC").ToObject(), NewStr("DEF").ToObject(), NewStr("GHI").ToObject(), NewStr("JKL").ToObject(), NewStr("MNO").ToObject(), NewStr("PQR").ToObject(), NewStr("STU").ToObject(), NewStr("VWX").ToObject(), NewStr("YZ)").ToObject(), NewStr("!@#").ToObject(), NewStr("$%^").ToObject(), NewStr("&*(").ToObject(), } var started, finished sync.WaitGroup stop := make(chan struct{}) runner := func(f func(*Frame, *Object, int)) { for i := 0; i < 8; i++ { started.Add(1) finished.Add(1) go func() { defer finished.Done() frame := NewRootFrame() i := 0 for _, k := range keys { f(frame, k, i) frame.RestoreExc(nil, nil) i++ } started.Done() for { if _, ok := <-stop; !ok { break } for _, k := range keys { f(frame, k, i) frame.RestoreExc(nil, nil) i++ } } }() } } d := NewDict().ToObject() runner(func(f *Frame, k *Object, _ int) { GetItem(f, d, k) }) runner(func(f *Frame, k *Object, i int) { mustNotRaise(nil, SetItem(f, d, k, NewInt(i).ToObject())) }) runner(func(f *Frame, k *Object, _ int) { DelItem(f, d, k) }) started.Wait() time.AfterFunc(time.Second, func() { close(stop) }) finished.Wait() } func newTestDict(elems ...interface{}) *Dict { if len(elems)%2 != 0 { panic("invalid test dict spec") } numItems := len(elems) / 2 d := NewDict() f := NewRootFrame() for i := 0; i < numItems; i++ { k := mustNotRaise(WrapNative(f, reflect.ValueOf(elems[i*2]))) v := mustNotRaise(WrapNative(f, reflect.ValueOf(elems[i*2+1]))) d.SetItem(f, k, v) } return d } ================================================ FILE: runtime/doc.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /* Package grumpy is the Grumpy runtime's Python API, analogous to CPython's C API. Data model All Python objects are represented by structs that are binary compatible with grumpy.Object, so for example the result of the Python expression "object()" is just an Object pointer. More complex primitive types like str and dict are represented by structs that augment Object by embedding it as their first field and holding other data in subsequent fields. These augmented structs can themselves be embedded for yet more complex types. Objects contain a pointer to their Python type, represented by grumpy.Type, and a pointer to their attribute dict, represented by grumpy.Dict. This dict may be nil as in the case of str or non-nil as in the case of type objects. Note that Grumpy objects do not have a refcount since Grumpy relies on Go's garbage collection to manage object lifetimes. Every Type object holds references to all its base classes as well as every class in its MRO list. Grumpy types also hold a reflect.Type instance known as the type's "basis". A type's basis represents the Go struct used to store instances of the type. It is an important invariant of the Grumpy runtime that an instance of a particular Python type is stored in the Go struct that is that type's basis. Violation of this invariant would mean that, for example, a str object could end up being stored in an unaugmented Object and accessing the str's value would access invalid memory. This invariant is enforced by Grumpy's API for primitive types and user defined classes. Upcasting and downcasting along the basis hierarchy is sometimes necessary, for example when passing a Str to a function accepting an Object. Upcasts are accomplished by accessing the embedded base type basis of the subclass, e.g. accessing the Object member of the Str struct. Downcasting requires unsafe.Pointer conversions. The safety of these conversions is guaranteed by the invariant discussed above. E.g. it is valid to cast an *Object with type StrType to a *Str because it was allocated with storage represented by StrType's basis, which is struct Str. Execution model User defined Python code blocks (modules, classes and functions) are implemented as Go closures with a state machine that allows the body of the block to be re-entered for exception handling, yield statements, etc. The generated code for the body of a code block looks something like this: 01: func(f *Frame) (*Object, *BaseException) { 02: switch (f.State()) { 03: case 0: goto Label0 04: case 1: goto Label1 05: ... 06: } 07: Label0: 08: ... 09: Label1: 10: ... 11: ... 12: } Frame is the basis type for Grumpy's "frame" objects and is very similar to CPython's type of the same name. The first argument f, therefore, represents a level in the Python stack. Upon entry into the body, the frame's state variable is checked and control jumps to the appropriate label. Upon first entry, the state variable will be 0 and the so execution will start at Label0. Later invocations may start at other labels. For example, an exception raised in the try block of a try/finally will cause the function above to return an exception as its second return value. The caller will then set state to the label corresponding to the finally clause and call back into the body. Python exceptions are represented by the BaseException basis struct. Grumpy API functions and generated code blocks propagate exceptions by returning *BaseException as their last return value. Exceptions are raised with the Frame.Raise*() methods which create exception objects to be propagated and set the exc info indicator for the current frame stack, similar to CPython. Python except clauses down the stack can then handle the propagated exception. Each generated body function is owned by a Block struct that is very similar to CPython's code object. Each Block has a name (e.g. the class' name) and the filename where the Python code was defined. A block is invoked via the *Block.Exec method which pushes a new frame on the call stack and then repeatedly calls the body function. This interplay is depicted below: *Block.Exec --> +-+ | | block func |1| --> +-+ | | |2| | | <-- +-+ | | --> +-+ | | |2| | | <-- +-+ <-- +-+ 1. *Block.Exec repeatedly calls block function until finished or an unhandled exception is encountered 2. Dispatch switch passes control to appropriate part of block function and executes When the body returns with a nil exception, the accompanying value is the returned from the block. If an exception is returned then the "checkpoint stack" is examined. This data structure stores recovery points within body that need to be executed when an exception occurs. Expanding on the try/finally example above, when an exception is raised in the try clause, the finally checkpoint is popped off the stack and its value is assigned to state. Body then gets called again and control is passed to the finally label. To make things concrete, here is a block of code containing a try/finally: 01: try: 02: print "foo" 03: finally: 04: print "bar" The generated code for this sinippet would look something like this: 01: func(f *Frame) (*Object, *BaseException) { 02: switch state { 03: case 0: goto Label0 04: case 1: goto Label1 05: } 06: Label0: 07: // line 1: try: 08: f.PushCheckpoint(1) 09: // line 2: print foo 10: raised = Print(f, []*Object{NewStr("foo").ToObject()}) 11: if raised != nil { 12: return nil, raised 13: } 14: f.PopCheckpoint() 15: Label1: 16: exc, tb = πF.RestoreExc(nil, nil) 17: // line 4: print bar 18: raised = Print(f, []*Object{NewStr("bar").ToObject()}) 19: if raised != nil { 20: return nil, raised 21: } 22: if exc != nil { 24: return nil, f.Raise(exc, nil, tb) 24: } 25: return None, nil 26: } There are a few relevant things worth noting here: 1. Upon entering the try clause on line 8, a checkpoint pointing to Label1 (the finally clause) is pushed onto the stack. If the try clause does not raise, the checkpoint is popped on line 14 and control falls through to Label1 without having to re-enter the body function. 2. Lines 10 and 18 are the two print statements. Exceptions raised during execution of these statements are returned immediately. In general, Python statements map to one or more Grumpy API function calls which may propagate exceptions. 3. Control of the finally clause begins on line 16 where the exception indicator is cleared and its original value is stored and re-raised at the end of the clause. This matches CPython's behavior where exc info is cleared during the finally block. A stack is used to store checkpoints because checkpoints can be nested. Continuing the example above, the finally clause itself could be in an except handler, e.g.: 01: try: 02: try: 03: print "foo" 04: finally: 05: print "bar" 06: except SomeException: 07: print "baz" Once the finally clause completes, it re-raises the exception and control is passed to the except handler label because it's next in the checkpoint stack. If the exception is an instance of SomeException then execution continues within the except clause. If it is some other kind of exception then it will be returned and control will be passed to the caller to find another checkpoint or unwind the call stack. Call model Python callables are represented by the Function basis struct and the corresponding Python "function" type. As in CPython, class methods and global functions are instances of this type. Associated with each instance is a Go function with the signature: func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) The args slice and kwargs dict contain the positional and keyword arguments provided by the caller. Both builtin functions and those in user defined Python code are called using this convention, however the latter are wrapped in a layer represented by the FunctionSpec struct that validates arguments and substitutes absent keyword parameters with their default. Once the spec is validated, it passes control to the spec function: func(f *Frame, args []*Object) (*Object, *BaseException) Here, the args slice contains an element for each argument present in the Python function's parameter list, in the same order. Every value is non-nil since default values have been substituted where necessary by the function spec. If parameters with the * or ** specifiers are present in the function signature, they are the last element(s) in args and hold any extra positional or keyword arguments provided by the caller. Generated code within the spec function consists of three main parts: +----------------------+ | Spec func | | --------- | | Declare locals | | Declare temporaries | | +------------------+ | | | Body func | | | | ---------- | | | | Dispatch switch | | | | Labels | | | +------------------+ | | Block.Exec(body) | +----------------------+ Locals and temporaries are defined as local variables at the top of the spec function. Below that, the body function is defined which is stateless except for what it inherits from its enclosing scope and from the passed frame. This is important because the body function will be repeatedly reenetered, but all of the state will have a lifetime longer than any particular invocation because it belongs to the spec function's scope. Finally, *Block.Exec is called which drives the state machine, calling into the body function as appropriate. Generator functions work much the same way except that instead of calling Exec on the block directly, the block is returned and the generator's next() method calls Exec until its contents are exhausted. */ package grumpy ================================================ FILE: runtime/exceptions.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy var ( // ArithmeticErrorType corresponds to the Python type 'ArithmeticError'. ArithmeticErrorType = newSimpleType("ArithmeticError", StandardErrorType) // AssertionErrorType corresponds to the Python type 'AssertionError'. AssertionErrorType = newSimpleType("AssertionError", StandardErrorType) // AttributeErrorType corresponds to the Python type 'AttributeError'. AttributeErrorType = newSimpleType("AttributeError", StandardErrorType) // BytesWarningType corresponds to the Python type 'BytesWarning'. BytesWarningType = newSimpleType("BytesWarning", WarningType) // DeprecationWarningType corresponds to the Python type 'DeprecationWarning'. DeprecationWarningType = newSimpleType("DeprecationWarning", WarningType) // EnvironmentErrorType corresponds to the Python type // 'EnvironmentError'. EnvironmentErrorType = newSimpleType("EnvironmentError", StandardErrorType) // EOFErrorType corresponds to the Python type 'EOFError'. EOFErrorType = newSimpleType("EOFError", StandardErrorType) // ExceptionType corresponds to the Python type 'Exception'. ExceptionType = newSimpleType("Exception", BaseExceptionType) // FutureWarningType corresponds to the Python type 'FutureWarning'. FutureWarningType = newSimpleType("FutureWarning", WarningType) // ImportErrorType corresponds to the Python type 'ImportError'. ImportErrorType = newSimpleType("ImportError", StandardErrorType) // ImportWarningType corresponds to the Python type 'ImportWarning'. ImportWarningType = newSimpleType("ImportWarning", WarningType) // IndexErrorType corresponds to the Python type 'IndexError'. IndexErrorType = newSimpleType("IndexError", LookupErrorType) // IOErrorType corresponds to the Python type 'IOError'. IOErrorType = newSimpleType("IOError", EnvironmentErrorType) // KeyboardInterruptType corresponds to the Python type 'KeyboardInterrupt'. KeyboardInterruptType = newSimpleType("KeyboardInterrupt", BaseExceptionType) // KeyErrorType corresponds to the Python type 'KeyError'. KeyErrorType = newSimpleType("KeyError", LookupErrorType) // LookupErrorType corresponds to the Python type 'LookupError'. LookupErrorType = newSimpleType("LookupError", StandardErrorType) // MemoryErrorType corresponds to the Python type 'MemoryError'. MemoryErrorType = newSimpleType("MemoryError", StandardErrorType) // NameErrorType corresponds to the Python type 'NameError'. NameErrorType = newSimpleType("NameError", StandardErrorType) // NotImplementedErrorType corresponds to the Python type // 'NotImplementedError'. NotImplementedErrorType = newSimpleType("NotImplementedError", RuntimeErrorType) // OSErrorType corresponds to the Python type 'OSError'. OSErrorType = newSimpleType("OSError", EnvironmentErrorType) // OverflowErrorType corresponds to the Python type 'OverflowError'. OverflowErrorType = newSimpleType("OverflowError", ArithmeticErrorType) // PendingDeprecationWarningType corresponds to the Python type 'PendingDeprecationWarning'. PendingDeprecationWarningType = newSimpleType("PendingDeprecationWarning", WarningType) // ReferenceErrorType corresponds to the Python type 'ReferenceError'. ReferenceErrorType = newSimpleType("ReferenceError", StandardErrorType) // RuntimeErrorType corresponds to the Python type 'RuntimeError'. RuntimeErrorType = newSimpleType("RuntimeError", StandardErrorType) // RuntimeWarningType corresponds to the Python type 'RuntimeWarning'. RuntimeWarningType = newSimpleType("RuntimeWarning", WarningType) // StandardErrorType corresponds to the Python type 'StandardError'. StandardErrorType = newSimpleType("StandardError", ExceptionType) // StopIterationType corresponds to the Python type 'StopIteration'. StopIterationType = newSimpleType("StopIteration", ExceptionType) // SyntaxErrorType corresponds to the Python type 'SyntaxError'. SyntaxErrorType = newSimpleType("SyntaxError", StandardErrorType) // SyntaxWarningType corresponds to the Python type 'SyntaxWarning'. SyntaxWarningType = newSimpleType("SyntaxWarning", WarningType) // SystemErrorType corresponds to the Python type 'SystemError'. SystemErrorType = newSimpleType("SystemError", StandardErrorType) // SystemExitType corresponds to the Python type 'SystemExit'. SystemExitType = newSimpleType("SystemExit", BaseExceptionType) // TypeErrorType corresponds to the Python type 'TypeError'. TypeErrorType = newSimpleType("TypeError", StandardErrorType) // UnboundLocalErrorType corresponds to the Python type // 'UnboundLocalError'. UnboundLocalErrorType = newSimpleType("UnboundLocalError", NameErrorType) // UnicodeDecodeErrorType corresponds to the Python type 'UnicodeDecodeError'. UnicodeDecodeErrorType = newSimpleType("UnicodeDecodeError", ValueErrorType) // UnicodeEncodeErrorType corresponds to the Python type 'UnicodeEncodeError'. UnicodeEncodeErrorType = newSimpleType("UnicodeEncodeError", ValueErrorType) // UnicodeErrorType corresponds to the Python type 'UnicodeError'. UnicodeErrorType = newSimpleType("UnicodeError", ValueErrorType) // UnicodeWarningType corresponds to the Python type 'UnicodeWarning'. UnicodeWarningType = newSimpleType("UnicodeWarning", WarningType) // UserWarningType corresponds to the Python type 'UserWarning'. UserWarningType = newSimpleType("UserWarning", WarningType) // ValueErrorType corresponds to the Python type 'ValueError'. ValueErrorType = newSimpleType("ValueError", StandardErrorType) // WarningType corresponds to the Python type 'Warning'. WarningType = newSimpleType("Warning", ExceptionType) // ZeroDivisionErrorType corresponds to the Python type // 'ZeroDivisionError'. ZeroDivisionErrorType = newSimpleType("ZeroDivisionError", ArithmeticErrorType) ) func systemExitInit(f *Frame, o *Object, args Args, kwargs KWArgs) (*Object, *BaseException) { baseExceptionInit(f, o, args, kwargs) code := None if len(args) > 0 { code = args[0] } if raised := SetAttr(f, o, NewStr("code"), code); raised != nil { return nil, raised } return None, nil } func initSystemExitType(map[string]*Object) { SystemExitType.slots.Init = &initSlot{systemExitInit} } ================================================ FILE: runtime/file.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "bufio" "bytes" "fmt" "io" "io/ioutil" "os" "reflect" "strings" "sync" ) // File represents Python 'file' objects. type File struct { Object // mutex synchronizes the state of the File struct, not access to the // underlying os.File. So, for example, when doing file reads and // writes we only acquire a read lock. mutex sync.Mutex mode string open bool Softspace int `attr:"softspace" attr_mode:"rw"` reader *bufio.Reader file *os.File skipNextLF bool univNewLine bool close *Object } // NewFileFromFD creates a file object from the given file descriptor fd. func NewFileFromFD(fd uintptr, close *Object) *File { // TODO: Use fcntl or something to get the mode of the descriptor. file := &File{ Object: Object{typ: FileType}, mode: "?", open: true, file: os.NewFile(fd, ""), } if close != None { file.close = close } file.reader = bufio.NewReader(file.file) return file } func toFileUnsafe(o *Object) *File { return (*File)(o.toPointer()) } func (f *File) name() string { name := "" if f.file != nil { name = f.file.Name() } return name } // ToObject upcasts f to an Object. func (f *File) ToObject() *Object { return &f.Object } func (f *File) readLine(maxBytes int) (string, error) { var buf bytes.Buffer numBytesRead := 0 for maxBytes < 0 || numBytesRead < maxBytes { b, err := f.reader.ReadByte() if err == io.EOF { break } if err != nil { return "", err } if b == '\r' && f.univNewLine { f.skipNextLF = true buf.WriteByte('\n') break } else if b == '\n' { if f.skipNextLF { f.skipNextLF = false continue // Do not increment numBytesRead. } else { buf.WriteByte(b) break } } else { buf.WriteByte(b) } numBytesRead++ } return buf.String(), nil } func (f *File) writeString(s string) error { f.mutex.Lock() defer f.mutex.Unlock() if !f.open { return io.ErrClosedPipe } if _, err := f.file.Write([]byte(s)); err != nil { return err } return nil } // FileType is the object representing the Python 'file' type. var FileType = newBasisType("file", reflect.TypeOf(File{}), toFileUnsafe, ObjectType) func fileInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) expectedTypes := []*Type{StrType, StrType} if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkFunctionArgs(f, "__init__", args, expectedTypes...); raised != nil { return nil, raised } mode := "r" if argc > 1 { mode = toStrUnsafe(args[1]).Value() } // TODO: Do something with the binary mode flag. var flag int switch mode { case "a", "ab": flag = os.O_WRONLY | os.O_CREATE | os.O_APPEND case "r", "rb", "rU", "U": flag = os.O_RDONLY case "r+", "r+b": flag = os.O_RDWR // Difference between r+ and a+ is that a+ automatically creates file. case "a+": flag = os.O_RDWR | os.O_CREATE | os.O_APPEND case "w+": flag = os.O_RDWR | os.O_CREATE case "w", "wb": flag = os.O_WRONLY | os.O_CREATE | os.O_TRUNC default: return nil, f.RaiseType(ValueErrorType, fmt.Sprintf("invalid mode string: %q", mode)) } file := toFileUnsafe(o) file.mutex.Lock() defer file.mutex.Unlock() osFile, err := os.OpenFile(toStrUnsafe(args[0]).Value(), flag, 0644) if err != nil { return nil, f.RaiseType(IOErrorType, err.Error()) } file.mode = mode file.open = true file.file = osFile file.reader = bufio.NewReader(osFile) file.univNewLine = strings.HasSuffix(mode, "U") return None, nil } func fileEnter(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__enter__", args, FileType); raised != nil { return nil, raised } return args[0], nil } func fileExit(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodVarArgs(f, "__exit__", args, FileType); raised != nil { return nil, raised } closeFunc, raised := GetAttr(f, args[0], NewStr("close"), nil) if raised != nil { return nil, raised } _, raised = closeFunc.Call(f, nil, nil) if raised != nil { return nil, raised } return None, nil } func fileClose(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "close", args, FileType); raised != nil { return nil, raised } file := toFileUnsafe(args[0]) file.mutex.Lock() defer file.mutex.Unlock() ret := None if file.open { var raised *BaseException if file.close != nil { ret, raised = file.close.Call(f, args, nil) } else if file.file != nil { if err := file.file.Close(); err != nil { raised = f.RaiseType(IOErrorType, err.Error()) } } if raised != nil { return nil, raised } } file.open = false return ret, nil } func fileClosed(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "closed", args, FileType); raised != nil { return nil, raised } file := toFileUnsafe(args[0]) file.mutex.Lock() c := !file.open file.mutex.Unlock() return GetBool(c).ToObject(), nil } func fileFileno(f *Frame, args Args, _ KWArgs) (ret *Object, raised *BaseException) { if raised := checkMethodArgs(f, "fileno", args, FileType); raised != nil { return nil, raised } file := toFileUnsafe(args[0]) file.mutex.Lock() if file.open { ret = NewInt(int(file.file.Fd())).ToObject() } else { raised = f.RaiseType(ValueErrorType, "I/O operation on closed file") } file.mutex.Unlock() return ret, raised } func fileGetName(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "_get_name", args, FileType); raised != nil { return nil, raised } file := toFileUnsafe(args[0]) file.mutex.Lock() name := file.name() file.mutex.Unlock() return NewStr(name).ToObject(), nil } func fileIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func fileNext(f *Frame, o *Object) (ret *Object, raised *BaseException) { file := toFileUnsafe(o) file.mutex.Lock() defer file.mutex.Unlock() if !file.open { return nil, f.RaiseType(ValueErrorType, "I/O operation on closed file") } line, err := file.readLine(-1) if err != nil { return nil, f.RaiseType(IOErrorType, err.Error()) } if line == "" { return nil, f.Raise(StopIterationType.ToObject(), nil, nil) } return NewStr(line).ToObject(), nil } func fileRead(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { file, size, raised := fileParseReadArgs(f, "read", args) if raised != nil { return nil, raised } file.mutex.Lock() defer file.mutex.Unlock() if !file.open { return nil, f.RaiseType(ValueErrorType, "I/O operation on closed file") } var data []byte var err error if size < 0 { data, err = ioutil.ReadAll(file.file) } else { data = make([]byte, size) var n int n, err = file.reader.Read(data) data = data[:n] } if err != nil && err != io.EOF { return nil, f.RaiseType(IOErrorType, err.Error()) } return NewStr(string(data)).ToObject(), nil } func fileReadLine(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { file, size, raised := fileParseReadArgs(f, "readline", args) if raised != nil { return nil, raised } file.mutex.Lock() defer file.mutex.Unlock() if !file.open { return nil, f.RaiseType(ValueErrorType, "I/O operation on closed file") } line, err := file.readLine(size) if err != nil { return nil, f.RaiseType(IOErrorType, err.Error()) } return NewStr(line).ToObject(), nil } func fileReadLines(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { // NOTE: The size hint behavior here is slightly different than // CPython. Here we read no more lines than necessary. In CPython a // minimum of 8KB or more will be read. file, size, raised := fileParseReadArgs(f, "readlines", args) if raised != nil { return nil, raised } file.mutex.Lock() defer file.mutex.Unlock() if !file.open { return nil, f.RaiseType(ValueErrorType, "I/O operation on closed file") } var lines []*Object numBytesRead := 0 for size < 0 || numBytesRead < size { line, err := file.readLine(-1) if err != nil { return nil, f.RaiseType(IOErrorType, err.Error()) } if line != "" { lines = append(lines, NewStr(line).ToObject()) } if !strings.HasSuffix(line, "\n") { break } numBytesRead += len(line) } return NewList(lines...).ToObject(), nil } func fileRepr(f *Frame, o *Object) (*Object, *BaseException) { file := toFileUnsafe(o) file.mutex.Lock() defer file.mutex.Unlock() var openState string if file.open { openState = "open" } else { openState = "closed" } var mode string if file.mode != "" { mode = file.mode } else { mode = "" } return NewStr(fmt.Sprintf("<%s file %q, mode %q at %p>", openState, file.name(), mode, file)).ToObject(), nil } func fileWrite(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "write", args, FileType, StrType); raised != nil { return nil, raised } file := toFileUnsafe(args[0]) file.mutex.Lock() defer file.mutex.Unlock() if !file.open { return nil, f.RaiseType(ValueErrorType, "I/O operation on closed file") } if _, err := file.file.Write([]byte(toStrUnsafe(args[1]).Value())); err != nil { return nil, f.RaiseType(IOErrorType, err.Error()) } return None, nil } func initFileType(dict map[string]*Object) { // TODO: Make enter/exit into slots. dict["__enter__"] = newBuiltinFunction("__enter__", fileEnter).ToObject() dict["__exit__"] = newBuiltinFunction("__exit__", fileExit).ToObject() dict["close"] = newBuiltinFunction("close", fileClose).ToObject() dict["closed"] = newBuiltinFunction("closed", fileClosed).ToObject() dict["fileno"] = newBuiltinFunction("fileno", fileFileno).ToObject() dict["name"] = newProperty(newBuiltinFunction("_get_name", fileGetName).ToObject(), nil, nil).ToObject() dict["read"] = newBuiltinFunction("read", fileRead).ToObject() dict["readline"] = newBuiltinFunction("readline", fileReadLine).ToObject() dict["readlines"] = newBuiltinFunction("readlines", fileReadLines).ToObject() dict["write"] = newBuiltinFunction("write", fileWrite).ToObject() FileType.slots.Init = &initSlot{fileInit} FileType.slots.Iter = &unaryOpSlot{fileIter} FileType.slots.Next = &unaryOpSlot{fileNext} FileType.slots.Repr = &unaryOpSlot{fileRepr} } func fileParseReadArgs(f *Frame, method string, args Args) (*File, int, *BaseException) { expectedTypes := []*Type{FileType, ObjectType} argc := len(args) if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkMethodArgs(f, method, args, expectedTypes...); raised != nil { return nil, 0, raised } size := -1 if argc > 1 { o, raised := IntType.Call(f, args[1:], nil) if raised != nil { return nil, 0, raised } size = toIntUnsafe(o).Value() } return toFileUnsafe(args[0]), size, nil } var ( // Stdin is an alias for sys.stdin. Stdin = NewFileFromFD(os.Stdin.Fd(), nil) // Stdout is an alias for sys.stdout. Stdout = NewFileFromFD(os.Stdout.Fd(), nil) // Stderr is an alias for sys.stderr. Stderr = NewFileFromFD(os.Stderr.Fd(), nil) ) ================================================ FILE: runtime/file_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "io/ioutil" "os" "regexp" "testing" ) var ( wantFileOperationRead = 1 wantFileOperationWrite = 2 ) func TestFileInit(t *testing.T) { f := newTestFile("blah blah") defer f.cleanup() cases := []invokeTestCase{ {args: wrapArgs(newObject(FileType), f.path), want: None}, {args: wrapArgs(newObject(FileType)), wantExc: mustCreateException(TypeErrorType, "'__init__' requires 2 arguments")}, {args: wrapArgs(newObject(FileType), f.path, "abc"), wantExc: mustCreateException(ValueErrorType, `invalid mode string: "abc"`)}, {args: wrapArgs(newObject(FileType), "nonexistent-file"), wantExc: mustCreateException(IOErrorType, "open nonexistent-file: no such file or directory")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(FileType, "__init__", &cas); err != "" { t.Error(err) } } } func TestFileClosed(t *testing.T) { f := newTestFile("foo\nbar") defer f.cleanup() closedFile := f.open("r") // This puts the file into an invalid state since Grumpy thinks // it's open even though the underlying file was closed. closedFile.file.Close() cases := []invokeTestCase{ {args: wrapArgs(newObject(FileType)), want: True.ToObject()}, {args: wrapArgs(f.open("r")), want: False.ToObject()}, {args: wrapArgs(closedFile), want: False.ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(FileType, "closed", &cas); err != "" { t.Error(err) } } } func TestFileCloseExit(t *testing.T) { f := newTestFile("foo\nbar") defer f.cleanup() for _, method := range []string{"close", "__exit__"} { closedFile := f.open("r") // This puts the file into an invalid state since Grumpy thinks // it's open even though the underlying file was closed. closedFile.file.Close() cases := []invokeTestCase{ {args: wrapArgs(newObject(FileType)), want: None}, {args: wrapArgs(f.open("r")), want: None}, {args: wrapArgs(closedFile), wantExc: mustCreateException(IOErrorType, closedFile.file.Close().Error())}, } for _, cas := range cases { if err := runInvokeMethodTestCase(FileType, method, &cas); err != "" { t.Error(err) } } } } func TestFileGetName(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, file *File) (*Object, *BaseException) { return GetAttr(f, file.ToObject(), NewStr("name"), nil) }) foo := newTestFile("foo") defer foo.cleanup() cases := []invokeTestCase{ {args: wrapArgs(foo.open("r")), want: NewStr(foo.path).ToObject()}, {args: wrapArgs(newObject(FileType)), want: NewStr("").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestFileIter(t *testing.T) { files := makeTestFiles() defer files.cleanup() closedFile := files[0].open("r") closedFile.file.Close() _, closedFileReadError := closedFile.file.Read(make([]byte, 10)) cases := []invokeTestCase{ {args: wrapArgs(files[0].open("r")), want: newTestList("foo").ToObject()}, {args: wrapArgs(files[0].open("rU")), want: newTestList("foo").ToObject()}, {args: wrapArgs(files[1].open("r")), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[1].open("rU")), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[2].open("r")), want: newTestList("foo\n", "bar").ToObject()}, {args: wrapArgs(files[2].open("rU")), want: newTestList("foo\n", "bar").ToObject()}, {args: wrapArgs(files[3].open("r")), want: newTestList("foo\r\n").ToObject()}, {args: wrapArgs(files[3].open("rU")), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[4].open("r")), want: newTestList("foo\rbar").ToObject()}, {args: wrapArgs(files[4].open("rU")), want: newTestList("foo\n", "bar").ToObject()}, {args: wrapArgs(closedFile), wantExc: mustCreateException(IOErrorType, closedFileReadError.Error())}, {args: wrapArgs(newObject(FileType)), wantExc: mustCreateException(ValueErrorType, "I/O operation on closed file")}, } for _, cas := range cases { if err := runInvokeTestCase(ListType.ToObject(), &cas); err != "" { t.Error(err) } } } func TestFileNext(t *testing.T) { files := makeTestFiles() defer files.cleanup() closedFile := files[0].open("r") closedFile.file.Close() _, closedFileReadError := closedFile.file.Read(make([]byte, 10)) cases := []invokeTestCase{ {args: wrapArgs(files[0].open("r")), want: NewStr("foo").ToObject()}, {args: wrapArgs(files[0].open("rU")), want: NewStr("foo").ToObject()}, {args: wrapArgs(files[1].open("r")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[1].open("rU")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[2].open("r")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[2].open("rU")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[3].open("r")), want: NewStr("foo\r\n").ToObject()}, {args: wrapArgs(files[3].open("rU")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[4].open("r")), want: NewStr("foo\rbar").ToObject()}, {args: wrapArgs(files[4].open("rU")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "unbound method next() must be called with file instance as first argument (got nothing instead)")}, {args: wrapArgs(closedFile), wantExc: mustCreateException(IOErrorType, closedFileReadError.Error())}, {args: wrapArgs(newObject(FileType)), wantExc: mustCreateException(ValueErrorType, "I/O operation on closed file")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(FileType, "next", &cas); err != "" { t.Error(err) } } } func TestFileRead(t *testing.T) { f := newTestFile("foo\nbar") defer f.cleanup() closedFile := f.open("r") closedFile.file.Close() _, closedFileReadError := closedFile.file.Read(make([]byte, 10)) cases := []invokeTestCase{ {args: wrapArgs(f.open("r")), want: NewStr("foo\nbar").ToObject()}, {args: wrapArgs(f.open("r"), 3), want: NewStr("foo").ToObject()}, {args: wrapArgs(f.open("r"), 1000), want: NewStr("foo\nbar").ToObject()}, {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "unbound method read() must be called with file instance as first argument (got nothing instead)")}, {args: wrapArgs(closedFile), wantExc: mustCreateException(IOErrorType, closedFileReadError.Error())}, {args: wrapArgs(newObject(FileType)), wantExc: mustCreateException(ValueErrorType, "I/O operation on closed file")}, {args: wrapArgs(newObject(FileType), "abc"), wantExc: mustCreateException(ValueErrorType, "invalid literal for int() with base 10: abc")}, {args: wrapArgs(newObject(FileType), 123, 456), wantExc: mustCreateException(TypeErrorType, "'read' of 'file' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(FileType, "read", &cas); err != "" { t.Error(err) } } } func TestFileReadLine(t *testing.T) { files := makeTestFiles() defer files.cleanup() closedFile := files[0].open("r") closedFile.file.Close() _, closedFileReadError := closedFile.file.Read(make([]byte, 10)) partialReadFile := files[5].open("rU") partialReadFile.readLine(-1) cases := []invokeTestCase{ {args: wrapArgs(files[0].open("r")), want: NewStr("foo").ToObject()}, {args: wrapArgs(files[0].open("rU")), want: NewStr("foo").ToObject()}, {args: wrapArgs(files[1].open("r")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[1].open("rU")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[2].open("r")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[2].open("rU")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[2].open("r"), 2), want: NewStr("fo").ToObject()}, {args: wrapArgs(files[2].open("r"), 3), want: NewStr("foo").ToObject()}, {args: wrapArgs(files[2].open("r"), 4), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[2].open("r"), 5), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[3].open("r")), want: NewStr("foo\r\n").ToObject()}, {args: wrapArgs(files[3].open("rU")), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[3].open("rU"), 3), want: NewStr("foo").ToObject()}, {args: wrapArgs(files[3].open("rU"), 4), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[3].open("rU"), 5), want: NewStr("foo\n").ToObject()}, {args: wrapArgs(files[4].open("r")), want: NewStr("foo\rbar").ToObject()}, {args: wrapArgs(files[4].open("rU")), want: NewStr("foo\n").ToObject()}, // Ensure that reading after a \r\n returns the requested // number of bytes when possible. Check that the trailing \n // does not count toward the bytes read. {args: wrapArgs(partialReadFile, 3), want: NewStr("bar").ToObject()}, {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "unbound method readline() must be called with file instance as first argument (got nothing instead)")}, {args: wrapArgs(closedFile), wantExc: mustCreateException(IOErrorType, closedFileReadError.Error())}, {args: wrapArgs(newObject(FileType)), wantExc: mustCreateException(ValueErrorType, "I/O operation on closed file")}, {args: wrapArgs(newObject(FileType), "abc"), wantExc: mustCreateException(ValueErrorType, "invalid literal for int() with base 10: abc")}, {args: wrapArgs(newObject(FileType), 123, 456), wantExc: mustCreateException(TypeErrorType, "'readline' of 'file' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(FileType, "readline", &cas); err != "" { t.Error(err) } } } func TestFileReadLines(t *testing.T) { files := makeTestFiles() defer files.cleanup() closedFile := files[0].open("r") closedFile.file.Close() _, closedFileReadError := closedFile.file.Read(make([]byte, 10)) partialReadFile := files[5].open("rU") partialReadFile.readLine(-1) cases := []invokeTestCase{ {args: wrapArgs(files[0].open("r")), want: newTestList("foo").ToObject()}, {args: wrapArgs(files[0].open("rU")), want: newTestList("foo").ToObject()}, {args: wrapArgs(files[1].open("r")), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[1].open("rU")), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[2].open("r")), want: newTestList("foo\n", "bar").ToObject()}, {args: wrapArgs(files[2].open("rU")), want: newTestList("foo\n", "bar").ToObject()}, {args: wrapArgs(files[2].open("r"), 2), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[2].open("r"), 3), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[2].open("r"), 4), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[2].open("r"), 5), want: newTestList("foo\n", "bar").ToObject()}, {args: wrapArgs(files[3].open("r")), want: newTestList("foo\r\n").ToObject()}, {args: wrapArgs(files[3].open("rU")), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[3].open("rU"), 3), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[3].open("rU"), 4), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[3].open("rU"), 5), want: newTestList("foo\n").ToObject()}, {args: wrapArgs(files[4].open("r")), want: newTestList("foo\rbar").ToObject()}, {args: wrapArgs(files[4].open("rU")), want: newTestList("foo\n", "bar").ToObject()}, // Ensure that reading after a \r\n returns the requested // number of bytes when possible. Check that the trailing \n // does not count toward the bytes read. {args: wrapArgs(partialReadFile, 3), want: newTestList("bar\n").ToObject()}, {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "unbound method readlines() must be called with file instance as first argument (got nothing instead)")}, {args: wrapArgs(closedFile), wantExc: mustCreateException(IOErrorType, closedFileReadError.Error())}, {args: wrapArgs(newObject(FileType)), wantExc: mustCreateException(ValueErrorType, "I/O operation on closed file")}, {args: wrapArgs(newObject(FileType), "abc"), wantExc: mustCreateException(ValueErrorType, "invalid literal for int() with base 10: abc")}, {args: wrapArgs(newObject(FileType), 123, 456), wantExc: mustCreateException(TypeErrorType, "'readlines' of 'file' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(FileType, "readlines", &cas); err != "" { t.Error(err) } } } func TestFileStrRepr(t *testing.T) { fun := newBuiltinFunction("TestFileStrRepr", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestFileStrRepr", args, ObjectType, StrType); raised != nil { return nil, raised } o := args[0] re := regexp.MustCompile(toStrUnsafe(args[1]).Value()) s, raised := ToStr(f, o) if raised != nil { return nil, raised } Assert(f, GetBool(re.MatchString(s.Value())).ToObject(), nil) s, raised = Repr(f, o) if raised != nil { return nil, raised } Assert(f, GetBool(re.MatchString(s.Value())).ToObject(), nil) return None, nil }).ToObject() f := newTestFile("foo\nbar") defer f.cleanup() closedFile := f.open("r").ToObject() mustNotRaise(fileClose(NewRootFrame(), []*Object{closedFile}, nil)) // Open a file for write. writeFile := f.open("wb") cases := []invokeTestCase{ {args: wrapArgs(f.open("r"), `^$`), want: None}, {args: wrapArgs(writeFile, `^$`), want: None}, {args: wrapArgs(newObject(FileType), `^", mode "" at \w+>$`), want: None}, {args: wrapArgs(closedFile, `^$`), want: None}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestFileWrite(t *testing.T) { fun := newBuiltinFunction("TestFileWrite", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "TestFileWrite", args, StrType, StrType, StrType); raised != nil { return nil, raised } writeFile, raised := FileType.Call(f, args[:2], nil) if raised != nil { return nil, raised } write, raised := GetAttr(f, writeFile, NewStr("write"), nil) if raised != nil { return nil, raised } if _, raised := write.Call(f, args[2:], nil); raised != nil { return nil, raised } contents, err := ioutil.ReadFile(toStrUnsafe(args[0]).Value()) if err != nil { return nil, f.RaiseType(RuntimeErrorType, fmt.Sprintf("error reading file: %s", err.Error())) } return NewStr(string(contents)).ToObject(), nil }).ToObject() // Create a temporary directory and cd to it. dir, err := ioutil.TempDir("", "TestFileWrite") if err != nil { t.Fatalf("failed to create temp dir: %v", err) } defer os.RemoveAll(dir) oldWd, err := os.Getwd() if err != nil { t.Fatalf("Getwd() failed: %s", err) } if err := os.Chdir(dir); err != nil { t.Fatalf("Chdir(%q) failed: %s", dir, err) } defer os.Chdir(oldWd) for _, filename := range []string{"truncate.txt", "readonly.txt", "append.txt", "rplus.txt", "aplus.txt", "wplus.txt"} { if err := ioutil.WriteFile(filename, []byte(filename), 0644); err != nil { t.Fatalf("ioutil.WriteFile(%q) failed: %s", filename, err) } } cases := []invokeTestCase{ {args: wrapArgs("noexist.txt", "w", "foo\nbar"), want: NewStr("foo\nbar").ToObject()}, {args: wrapArgs("truncate.txt", "w", "new contents"), want: NewStr("new contents").ToObject()}, {args: wrapArgs("append.txt", "a", "\nbar"), want: NewStr("append.txt\nbar").ToObject()}, {args: wrapArgs("rplus.txt", "r+", "fooey"), want: NewStr("fooey.txt").ToObject()}, {args: wrapArgs("noexistplus1.txt", "r+", "pooey"), wantExc: mustCreateException(IOErrorType, "open noexistplus1.txt: no such file or directory")}, {args: wrapArgs("aplus.txt", "a+", "\napper"), want: NewStr("aplus.txt\napper").ToObject()}, {args: wrapArgs("noexistplus3.txt", "a+", "snappbacktoreality"), want: NewStr("snappbacktoreality").ToObject()}, {args: wrapArgs("wplus.txt", "w+", "destructo"), want: NewStr("destructo").ToObject()}, {args: wrapArgs("noexistplus2.txt", "w+", "wapper"), want: NewStr("wapper").ToObject()}, {args: wrapArgs("readonly.txt", "r", "foo"), wantExc: mustCreateException(IOErrorType, "write readonly.txt: bad file descriptor")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } type testFile struct { path string files []*File } func newTestFile(contents string) *testFile { osFile, err := ioutil.TempFile("", "") if err != nil { panic(err) } if _, err := osFile.WriteString(contents); err != nil { panic(err) } if err := osFile.Close(); err != nil { panic(err) } return &testFile{path: osFile.Name()} } func (f *testFile) cleanup() { for _, file := range f.files { file.file.Close() } os.Remove(f.path) } func (f *testFile) open(mode string) *File { args := wrapArgs(f.path, mode) o := mustNotRaise(FileType.Call(NewRootFrame(), args, nil)) if o == nil || !o.isInstance(FileType) { panic(fmt.Sprintf("file%v = %v, want file object", args, o)) } file := toFileUnsafe(o) f.files = append(f.files, file) return file } type testFileSlice []*testFile func makeTestFiles() testFileSlice { return []*testFile{ newTestFile("foo"), newTestFile("foo\n"), newTestFile("foo\nbar"), newTestFile("foo\r\n"), newTestFile("foo\rbar"), newTestFile("foo\r\nbar\r\nbaz"), } } func (files testFileSlice) cleanup() { for _, f := range files { f.cleanup() } } ================================================ FILE: runtime/float.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "math" "math/big" "reflect" "strconv" "strings" "sync/atomic" "unicode" "unsafe" ) // FloatType is the object representing the Python 'float' type. var FloatType = newBasisType("float", reflect.TypeOf(Float{}), toFloatUnsafe, ObjectType) // Float represents Python 'float' objects. type Float struct { Object value float64 hash int } // NewFloat returns a new Float holding the given floating point value. func NewFloat(value float64) *Float { return &Float{Object: Object{typ: FloatType}, value: value} } func toFloatUnsafe(o *Object) *Float { return (*Float)(o.toPointer()) } // ToObject upcasts f to an Object. func (f *Float) ToObject() *Object { return &f.Object } // Value returns the underlying floating point value held by f. func (f *Float) Value() float64 { return f.value } func floatAbs(f *Frame, o *Object) (*Object, *BaseException) { z := toFloatUnsafe(o).Value() return NewFloat(math.Abs(z)).ToObject(), nil } func floatAdd(f *Frame, v, w *Object) (*Object, *BaseException) { return floatArithmeticOp(f, "__add__", v, w, func(v, w float64) float64 { return v + w }) } func floatDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return floatDivModOp(f, "__div__", v, w, func(v, w float64) (float64, bool) { if w == 0.0 { return 0, false } return v / w, true }) } func floatDivMod(f *Frame, v, w *Object) (*Object, *BaseException) { return floatDivAndModOp(f, "__divmod__", v, w, func(v, w float64) (float64, float64, bool) { m, r := floatModFunc(v, w) if !r { return 0, 0, false } return math.Floor(v / w), m, true }) } func floatEq(f *Frame, v, w *Object) (*Object, *BaseException) { return floatCompare(toFloatUnsafe(v), w, False, True, False), nil } func floatFloat(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func floatFloorDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return floatDivModOp(f, "__floordiv__", v, w, func(v, w float64) (float64, bool) { if w == 0.0 { return 0, false } return math.Floor(v / w), true }) } func floatGE(f *Frame, v, w *Object) (*Object, *BaseException) { return floatCompare(toFloatUnsafe(v), w, False, True, True), nil } func floatGetNewArgs(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__getnewargs__", args, FloatType); raised != nil { return nil, raised } return NewTuple1(args[0]).ToObject(), nil } func floatGT(f *Frame, v, w *Object) (*Object, *BaseException) { return floatCompare(toFloatUnsafe(v), w, False, False, True), nil } func floatHash(f *Frame, o *Object) (*Object, *BaseException) { v := toFloatUnsafe(o) p := (*unsafe.Pointer)(unsafe.Pointer(&v.hash)) if lp := atomic.LoadPointer(p); lp != unsafe.Pointer(nil) { return (*Int)(lp).ToObject(), nil } hash := hashFloat(v.Value()) if hash == -1 { hash-- } h := NewInt(hash) atomic.StorePointer(p, unsafe.Pointer(h)) return h.ToObject(), nil } func floatInt(f *Frame, o *Object) (*Object, *BaseException) { val := toFloatUnsafe(o).Value() if math.IsInf(val, 0) { return nil, f.RaiseType(OverflowErrorType, "cannot convert float infinity to integer") } if math.IsNaN(val) { return nil, f.RaiseType(OverflowErrorType, "cannot convert float NaN to integer") } i := big.Int{} big.NewFloat(val).Int(&i) if !numInIntRange(&i) { return NewLong(&i).ToObject(), nil } return NewInt(int(i.Int64())).ToObject(), nil } func floatLong(f *Frame, o *Object) (*Object, *BaseException) { val := toFloatUnsafe(o).Value() if math.IsInf(val, 0) { return nil, f.RaiseType(OverflowErrorType, "cannot convert float infinity to integer") } if math.IsNaN(val) { return nil, f.RaiseType(OverflowErrorType, "cannot convert float NaN to integer") } i, _ := big.NewFloat(val).Int(nil) return NewLong(i).ToObject(), nil } func floatLE(f *Frame, v, w *Object) (*Object, *BaseException) { return floatCompare(toFloatUnsafe(v), w, True, True, False), nil } func floatLT(f *Frame, v, w *Object) (*Object, *BaseException) { return floatCompare(toFloatUnsafe(v), w, True, False, False), nil } func floatMod(f *Frame, v, w *Object) (*Object, *BaseException) { return floatDivModOp(f, "__mod__", v, w, floatModFunc) } func floatMul(f *Frame, v, w *Object) (*Object, *BaseException) { return floatArithmeticOp(f, "__mul__", v, w, func(v, w float64) float64 { return v * w }) } func floatNative(f *Frame, o *Object) (reflect.Value, *BaseException) { return reflect.ValueOf(toFloatUnsafe(o).Value()), nil } func floatNE(f *Frame, v, w *Object) (*Object, *BaseException) { return floatCompare(toFloatUnsafe(v), w, True, False, True), nil } func floatNeg(f *Frame, o *Object) (*Object, *BaseException) { z := toFloatUnsafe(o).Value() return NewFloat(-z).ToObject(), nil } func floatNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) if argc == 0 { return newObject(t), nil } if argc != 1 { return nil, f.RaiseType(TypeErrorType, "'__new__' of 'float' requires 0 or 1 arguments") } if t != FloatType { // Allocate a plain float then copy it's value into an object // of the float subtype. x, raised := floatNew(f, FloatType, args, nil) if raised != nil { return nil, raised } result := toFloatUnsafe(newObject(t)) result.value = toFloatUnsafe(x).Value() return result.ToObject(), nil } o := args[0] if floatSlot := o.typ.slots.Float; floatSlot != nil { fl, raised := floatConvert(floatSlot, f, o) if raised != nil { return nil, raised } return fl.ToObject(), nil } if !o.isInstance(StrType) { return nil, f.RaiseType(TypeErrorType, "float() argument must be a string or a number") } s := toStrUnsafe(o).Value() result, err := strconv.ParseFloat(s, 64) if err != nil { return nil, f.RaiseType(ValueErrorType, fmt.Sprintf("could not convert string to float: %s", s)) } return NewFloat(result).ToObject(), nil } func floatNonZero(f *Frame, o *Object) (*Object, *BaseException) { return GetBool(toFloatUnsafe(o).Value() != 0).ToObject(), nil } func floatPos(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func floatPow(f *Frame, v, w *Object) (*Object, *BaseException) { return floatArithmeticOp(f, "__pow__", v, w, func(v, w float64) float64 { return math.Pow(v, w) }) } func floatRAdd(f *Frame, v, w *Object) (*Object, *BaseException) { return floatArithmeticOp(f, "__radd__", v, w, func(v, w float64) float64 { return w + v }) } func floatRDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return floatDivModOp(f, "__rdiv__", v, w, func(v, w float64) (float64, bool) { if v == 0.0 { return 0, false } return w / v, true }) } func floatRDivMod(f *Frame, v, w *Object) (*Object, *BaseException) { return floatDivAndModOp(f, "__rdivmod__", v, w, func(v, w float64) (float64, float64, bool) { m, r := floatModFunc(w, v) if !r { return 0, 0, false } return w / v, m, true }) } const ( floatReprPrecision = 16 floatStrPrecision = 12 ) func floatRepr(f *Frame, o *Object) (*Object, *BaseException) { return NewStr(floatToString(toFloatUnsafe(o).Value(), floatReprPrecision)).ToObject(), nil } func floatRFloorDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return floatDivModOp(f, "__rfloordiv__", v, w, func(v, w float64) (float64, bool) { if v == 0.0 { return 0, false } return math.Floor(w / v), true }) } func floatRMod(f *Frame, v, w *Object) (*Object, *BaseException) { return floatDivModOp(f, "__rmod__", v, w, func(v, w float64) (float64, bool) { return floatModFunc(w, v) }) } func floatRMul(f *Frame, v, w *Object) (*Object, *BaseException) { return floatArithmeticOp(f, "__rmul__", v, w, func(v, w float64) float64 { return w * v }) } func floatRPow(f *Frame, v, w *Object) (*Object, *BaseException) { return floatArithmeticOp(f, "__rpow", v, w, func(v, w float64) float64 { return math.Pow(w, v) }) } func floatRSub(f *Frame, v, w *Object) (*Object, *BaseException) { return floatArithmeticOp(f, "__rsub__", v, w, func(v, w float64) float64 { return w - v }) } func floatStr(f *Frame, o *Object) (*Object, *BaseException) { return NewStr(floatToString(toFloatUnsafe(o).Value(), floatStrPrecision)).ToObject(), nil } func floatSub(f *Frame, v, w *Object) (*Object, *BaseException) { return floatArithmeticOp(f, "__sub__", v, w, func(v, w float64) float64 { return v - w }) } func initFloatType(dict map[string]*Object) { dict["__getnewargs__"] = newBuiltinFunction("__getnewargs__", floatGetNewArgs).ToObject() FloatType.slots.Abs = &unaryOpSlot{floatAbs} FloatType.slots.Add = &binaryOpSlot{floatAdd} FloatType.slots.Div = &binaryOpSlot{floatDiv} FloatType.slots.DivMod = &binaryOpSlot{floatDivMod} FloatType.slots.Eq = &binaryOpSlot{floatEq} FloatType.slots.Float = &unaryOpSlot{floatFloat} FloatType.slots.FloorDiv = &binaryOpSlot{floatFloorDiv} FloatType.slots.GE = &binaryOpSlot{floatGE} FloatType.slots.GT = &binaryOpSlot{floatGT} FloatType.slots.Hash = &unaryOpSlot{floatHash} FloatType.slots.Int = &unaryOpSlot{floatInt} FloatType.slots.Long = &unaryOpSlot{floatLong} FloatType.slots.LE = &binaryOpSlot{floatLE} FloatType.slots.LT = &binaryOpSlot{floatLT} FloatType.slots.Mod = &binaryOpSlot{floatMod} FloatType.slots.Mul = &binaryOpSlot{floatMul} FloatType.slots.Native = &nativeSlot{floatNative} FloatType.slots.NE = &binaryOpSlot{floatNE} FloatType.slots.Neg = &unaryOpSlot{floatNeg} FloatType.slots.New = &newSlot{floatNew} FloatType.slots.NonZero = &unaryOpSlot{floatNonZero} FloatType.slots.Pos = &unaryOpSlot{floatPos} FloatType.slots.Pow = &binaryOpSlot{floatPow} FloatType.slots.RAdd = &binaryOpSlot{floatRAdd} FloatType.slots.RDiv = &binaryOpSlot{floatRDiv} FloatType.slots.RDivMod = &binaryOpSlot{floatRDivMod} FloatType.slots.Repr = &unaryOpSlot{floatRepr} FloatType.slots.RFloorDiv = &binaryOpSlot{floatRFloorDiv} FloatType.slots.RMod = &binaryOpSlot{floatRMod} FloatType.slots.RMul = &binaryOpSlot{floatRMul} FloatType.slots.RPow = &binaryOpSlot{floatRPow} FloatType.slots.RSub = &binaryOpSlot{floatRSub} FloatType.slots.Str = &unaryOpSlot{floatStr} FloatType.slots.Sub = &binaryOpSlot{floatSub} } func floatArithmeticOp(f *Frame, method string, v, w *Object, fun func(v, w float64) float64) (*Object, *BaseException) { floatW, ok := floatCoerce(w) if !ok { if math.IsInf(floatW, 0) { return nil, f.RaiseType(OverflowErrorType, "long int too large to convert to float") } return NotImplemented, nil } return NewFloat(fun(toFloatUnsafe(v).Value(), floatW)).ToObject(), nil } func floatCompare(v *Float, w *Object, ltResult, eqResult, gtResult *Int) *Object { lhs := v.Value() rhs, ok := floatCoerce(w) if !ok { if !math.IsInf(rhs, 0) { return NotImplemented } // When floatCoerce returns (Inf, false) it indicates an // overflow - abs(rhs) is between MaxFloat64 and Inf. // When comparing with infinite floats, rhs might as well be 0. // Otherwise, let the compare proceed normally as |rhs| might // as well be infinite, since it's outside the range of finite // floats. if math.IsInf(lhs, 0) { rhs = 0 } } if lhs < rhs { return ltResult.ToObject() } if lhs == rhs { return eqResult.ToObject() } if lhs > rhs { return gtResult.ToObject() } // There must be a NaN involved, which always compares false, even to other NaNs. // This is true both in Go and in Python. return False.ToObject() } // floatCoerce will coerce any numeric type to a float. If all is // well, it will return the float64 value, and true (OK). If an overflow // occurs, it will return either (+Inf, false) or (-Inf, false) depending // on whether the source value was too large or too small. Note that if the // source number is an infinite float, the result will be infinite without // overflow, (+-Inf, true). // If the input is not a number, it will return (0, false). func floatCoerce(o *Object) (float64, bool) { switch { case o.isInstance(IntType): return float64(toIntUnsafe(o).Value()), true case o.isInstance(LongType): f, _ := new(big.Float).SetInt(toLongUnsafe(o).Value()).Float64() // If f is infinite, that indicates the big.Int was too large // or too small to be represented as a float64. In that case, // indicate the overflow by returning (f, false). overflow := math.IsInf(f, 0) return f, !overflow case o.isInstance(FloatType): return toFloatUnsafe(o).Value(), true default: return 0, false } } func floatConvert(floatSlot *unaryOpSlot, f *Frame, o *Object) (*Float, *BaseException) { result, raised := floatSlot.Fn(f, o) if raised != nil { return nil, raised } if !result.isInstance(FloatType) { exc := fmt.Sprintf("__float__ returned non-float (type %s)", result.typ.Name()) return nil, f.RaiseType(TypeErrorType, exc) } return toFloatUnsafe(result), nil } func floatDivModOp(f *Frame, method string, v, w *Object, fun func(v, w float64) (float64, bool)) (*Object, *BaseException) { floatW, ok := floatCoerce(w) if !ok { if math.IsInf(floatW, 0) { return nil, f.RaiseType(OverflowErrorType, "long int too large to convert to float") } return NotImplemented, nil } x, ok := fun(toFloatUnsafe(v).Value(), floatW) if !ok { return nil, f.RaiseType(ZeroDivisionErrorType, "float division or modulo by zero") } return NewFloat(x).ToObject(), nil } func floatDivAndModOp(f *Frame, method string, v, w *Object, fun func(v, w float64) (float64, float64, bool)) (*Object, *BaseException) { floatW, ok := floatCoerce(w) if !ok { if math.IsInf(floatW, 0) { return nil, f.RaiseType(OverflowErrorType, "long int too large to convert to float") } return NotImplemented, nil } q, m, ok := fun(toFloatUnsafe(v).Value(), floatW) if !ok { return nil, f.RaiseType(ZeroDivisionErrorType, "float division or modulo by zero") } return NewTuple2(NewFloat(q).ToObject(), NewFloat(m).ToObject()).ToObject(), nil } func hashFloat(v float64) int { if math.IsNaN(v) { return 0 } if math.IsInf(v, 0) { if math.IsInf(v, 1) { return 314159 } if math.IsInf(v, -1) { return -271828 } return 0 } _, fracPart := math.Modf(v) if fracPart == 0.0 { i := big.Int{} big.NewFloat(v).Int(&i) if numInIntRange(&i) { return int(i.Int64()) } // TODO: hashBigInt() is not yet matched that of cpython or pypy. return hashBigInt(&i) } v, expo := math.Frexp(v) v *= 2147483648.0 hiPart := int(v) v = (v - float64(hiPart)) * 2147483648.0 x := int(hiPart + int(v) + (expo << 15)) return x } func floatModFunc(v, w float64) (float64, bool) { if w == 0.0 { return 0, false } x := math.Mod(v, w) if x != 0 && math.Signbit(x) != math.Signbit(w) { // In Python the result of the modulo operator is // always the same sign as the divisor, whereas in Go, // the result is always the same sign as the dividend. // Therefore we need to do an adjustment when the sign // of the modulo result differs from that of the // divisor. x += w } return x, true } func floatToString(f float64, p int) string { s := unsignPositiveInf(strings.ToLower(strconv.FormatFloat(f, 'g', p, 64))) fun := func(r rune) bool { return !unicode.IsDigit(r) } if i := strings.IndexFunc(s, fun); i == -1 { s += ".0" } return s } func unsignPositiveInf(s string) string { if s == "+inf" { return "inf" } return s } ================================================ FILE: runtime/float_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "math" "math/big" "testing" ) var ( // bigLongNumber is a number that is too large to be converted to // a float64. bigLongNumber = new(big.Int).Exp(big.NewInt(10), big.NewInt(1000), nil) // biggestFloat is the largest integer that can be converted to a // Python long (in CPython) without overflow. // Its value is 2**1024 - 2**(1024-54) - 1. biggestFloat = func(z *big.Int) *big.Int { z.SetBit(z, 1024, 1) z.Sub(z, big.NewInt(1)) z.SetBit(z, 1024-54, 0) return z }(new(big.Int)) ) func TestFloatArithmeticOps(t *testing.T) { cases := []struct { fun func(f *Frame, v, w *Object) (*Object, *BaseException) v, w *Object want *Object wantExc *BaseException }{ {Add, NewFloat(1).ToObject(), NewFloat(1).ToObject(), NewFloat(2).ToObject(), nil}, {Add, NewFloat(1.5).ToObject(), NewInt(1).ToObject(), NewFloat(2.5).ToObject(), nil}, {Add, NewInt(1).ToObject(), NewFloat(1.5).ToObject(), NewFloat(2.5).ToObject(), nil}, {Add, NewFloat(1.7976931348623157e+308).ToObject(), NewFloat(1.7976931348623157e+308).ToObject(), NewFloat(math.Inf(1)).ToObject(), nil}, {Add, NewFloat(1.7976931348623157e+308).ToObject(), NewFloat(-1.7976931348623157e+308).ToObject(), NewFloat(0).ToObject(), nil}, {Add, NewFloat(math.Inf(1)).ToObject(), NewFloat(math.Inf(1)).ToObject(), NewFloat(math.Inf(1)).ToObject(), nil}, {Add, NewFloat(math.Inf(1)).ToObject(), NewFloat(math.Inf(-1)).ToObject(), NewFloat(math.NaN()).ToObject(), nil}, {Add, newObject(ObjectType), NewFloat(-1).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'object' and 'float'")}, {Div, NewFloat(12.5).ToObject(), NewFloat(4).ToObject(), NewFloat(3.125).ToObject(), nil}, {Div, NewFloat(-12.5).ToObject(), NewInt(4).ToObject(), NewFloat(-3.125).ToObject(), nil}, {Div, NewInt(25).ToObject(), NewFloat(5).ToObject(), NewFloat(5.0).ToObject(), nil}, {Div, NewFloat(math.Inf(1)).ToObject(), NewFloat(math.Inf(1)).ToObject(), NewFloat(math.NaN()).ToObject(), nil}, {Div, NewFloat(math.Inf(-1)).ToObject(), NewInt(-20).ToObject(), NewFloat(math.Inf(1)).ToObject(), nil}, {Div, NewInt(1).ToObject(), NewFloat(math.Inf(1)).ToObject(), NewFloat(0).ToObject(), nil}, {Div, newObject(ObjectType), NewFloat(1.1).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for /: 'object' and 'float'")}, {Div, True.ToObject(), NewFloat(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "float division or modulo by zero")}, {Div, NewFloat(math.Inf(1)).ToObject(), NewFloat(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "float division or modulo by zero")}, {Div, NewFloat(1.0).ToObject(), NewLong(bigLongNumber).ToObject(), nil, mustCreateException(OverflowErrorType, "long int too large to convert to float")}, {FloorDiv, NewFloat(12.5).ToObject(), NewFloat(4).ToObject(), NewFloat(3).ToObject(), nil}, {FloorDiv, NewFloat(-12.5).ToObject(), NewInt(4).ToObject(), NewFloat(-4).ToObject(), nil}, {FloorDiv, NewInt(25).ToObject(), NewFloat(5).ToObject(), NewFloat(5.0).ToObject(), nil}, {FloorDiv, NewFloat(math.Inf(1)).ToObject(), NewFloat(math.Inf(1)).ToObject(), NewFloat(math.NaN()).ToObject(), nil}, {FloorDiv, NewFloat(math.Inf(-1)).ToObject(), NewInt(-20).ToObject(), NewFloat(math.Inf(1)).ToObject(), nil}, {FloorDiv, NewInt(1).ToObject(), NewFloat(math.Inf(1)).ToObject(), NewFloat(0).ToObject(), nil}, {FloorDiv, newObject(ObjectType), NewFloat(1.1).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for //: 'object' and 'float'")}, {FloorDiv, NewFloat(1.0).ToObject(), NewLong(bigLongNumber).ToObject(), nil, mustCreateException(OverflowErrorType, "long int too large to convert to float")}, {FloorDiv, True.ToObject(), NewFloat(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "float division or modulo by zero")}, {FloorDiv, NewFloat(math.Inf(1)).ToObject(), NewFloat(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "float division or modulo by zero")}, {Mod, NewFloat(50.5).ToObject(), NewInt(10).ToObject(), NewFloat(0.5).ToObject(), nil}, {Mod, NewFloat(50.5).ToObject(), NewFloat(-10).ToObject(), NewFloat(-9.5).ToObject(), nil}, {Mod, NewFloat(-20.2).ToObject(), NewFloat(40).ToObject(), NewFloat(19.8).ToObject(), nil}, {Mod, NewFloat(math.Inf(1)).ToObject(), NewInt(10).ToObject(), NewFloat(math.NaN()).ToObject(), nil}, {Mod, NewInt(17).ToObject(), NewFloat(-4.25).ToObject(), NewFloat(0).ToObject(), nil}, {Mod, NewInt(10).ToObject(), NewFloat(-8).ToObject(), NewFloat(-6).ToObject(), nil}, {Mod, NewFloat(4.5).ToObject(), NewFloat(math.Inf(1)).ToObject(), NewFloat(4.5).ToObject(), nil}, {Mod, NewFloat(4.5).ToObject(), NewFloat(math.Inf(-1)).ToObject(), NewFloat(math.Inf(-1)).ToObject(), nil}, {Mod, NewFloat(math.Inf(1)).ToObject(), NewFloat(math.Inf(-1)).ToObject(), NewFloat(math.NaN()).ToObject(), nil}, {Mod, None, NewFloat(42).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for %: 'NoneType' and 'float'")}, {Mod, NewFloat(-32.25).ToObject(), NewInt(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "float division or modulo by zero")}, {Mod, NewFloat(math.Inf(-1)).ToObject(), NewFloat(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "float division or modulo by zero")}, {Mod, NewInt(2).ToObject(), NewFloat(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "float division or modulo by zero")}, {Mul, NewFloat(1.2).ToObject(), True.ToObject(), NewFloat(1.2).ToObject(), nil}, {Mul, NewInt(-4).ToObject(), NewFloat(1.2).ToObject(), NewFloat(-4.8).ToObject(), nil}, {Mul, NewFloat(math.Inf(1)).ToObject(), NewInt(-5).ToObject(), NewFloat(math.Inf(-1)).ToObject(), nil}, {Mul, False.ToObject(), NewFloat(math.Inf(1)).ToObject(), NewFloat(math.NaN()).ToObject(), nil}, {Mul, None, NewFloat(1.5).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'NoneType' and 'float'")}, {Pow, NewFloat(2.0).ToObject(), NewInt(10).ToObject(), NewFloat(1024.0).ToObject(), nil}, {Pow, NewFloat(2.0).ToObject(), NewFloat(-2.0).ToObject(), NewFloat(0.25).ToObject(), nil}, {Pow, newObject(ObjectType), NewFloat(2.0).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for **: 'object' and 'float'")}, {Pow, NewFloat(2.0).ToObject(), newObject(ObjectType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for **: 'float' and 'object'")}, {Sub, NewFloat(21.3).ToObject(), NewFloat(35.6).ToObject(), NewFloat(-14.3).ToObject(), nil}, {Sub, True.ToObject(), NewFloat(1.5).ToObject(), NewFloat(-0.5).ToObject(), nil}, {Sub, NewFloat(1.0).ToObject(), NewList().ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for -: 'float' and 'list'")}, {Sub, NewFloat(math.Inf(1)).ToObject(), NewFloat(math.Inf(1)).ToObject(), NewFloat(math.NaN()).ToObject(), nil}, } for _, cas := range cases { switch got, result := checkInvokeResult(wrapFuncForTest(cas.fun), []*Object{cas.v, cas.w}, cas.want, cas.wantExc); result { case checkInvokeResultExceptionMismatch: t.Errorf("%s(%v, %v) raised %v, want %v", getFuncName(cas.fun), cas.v, cas.w, got, cas.wantExc) case checkInvokeResultReturnValueMismatch: // Handle NaN specially, since NaN != NaN. if got == nil || cas.want == nil || !got.isInstance(FloatType) || !cas.want.isInstance(FloatType) || !math.IsNaN(toFloatUnsafe(got).Value()) || !math.IsNaN(toFloatUnsafe(cas.want).Value()) { t.Errorf("%s(%v, %v) = %v, want %v", getFuncName(cas.fun), cas.v, cas.w, got, cas.want) } } } } func TestFloatDivMod(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(12.5, 4.0), want: NewTuple2(NewFloat(3).ToObject(), NewFloat(0.5).ToObject()).ToObject()}, {args: wrapArgs(-12.5, 4.0), want: NewTuple2(NewFloat(-4).ToObject(), NewFloat(3.5).ToObject()).ToObject()}, {args: wrapArgs(25.0, 5.0), want: NewTuple2(NewFloat(5).ToObject(), NewFloat(0).ToObject()).ToObject()}, {args: wrapArgs(-20.2, 40.0), want: NewTuple2(NewFloat(-1).ToObject(), NewFloat(19.8).ToObject()).ToObject()}, {args: wrapArgs(math.Inf(1), math.Inf(1)), want: NewTuple2(NewFloat(math.NaN()).ToObject(), NewFloat(math.NaN()).ToObject()).ToObject()}, {args: wrapArgs(math.Inf(1), math.Inf(-1)), want: NewTuple2(NewFloat(math.NaN()).ToObject(), NewFloat(math.NaN()).ToObject()).ToObject()}, {args: wrapArgs(math.Inf(-1), -20.0), want: NewTuple2(NewFloat(math.Inf(1)).ToObject(), NewFloat(math.NaN()).ToObject()).ToObject()}, {args: wrapArgs(1, math.Inf(1)), want: NewTuple2(NewFloat(0).ToObject(), NewFloat(1).ToObject()).ToObject()}, {args: wrapArgs(newObject(ObjectType), 1.1), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for divmod(): 'object' and 'float'")}, {args: wrapArgs(True.ToObject(), 0.0), wantExc: mustCreateException(ZeroDivisionErrorType, "float division or modulo by zero")}, {args: wrapArgs(math.Inf(1), 0.0), wantExc: mustCreateException(ZeroDivisionErrorType, "float division or modulo by zero")}, {args: wrapArgs(1.0, bigLongNumber), wantExc: mustCreateException(OverflowErrorType, "long int too large to convert to float")}, } for _, cas := range cases { switch got, result := checkInvokeResult(wrapFuncForTest(DivMod), cas.args, cas.want, cas.wantExc); result { case checkInvokeResultExceptionMismatch: t.Errorf("float.__divmod__%v raised %v, want %v", cas.args, got, cas.wantExc) case checkInvokeResultReturnValueMismatch: // Handle NaN specially, since NaN != NaN. if got == nil || cas.want == nil || !got.isInstance(TupleType) || !cas.want.isInstance(TupleType) || !isNaNTupleFloat(got, cas.want) { t.Errorf("float.__divmod__%v = %v, want %v", cas.args, got, cas.want) } } } } func isNaNTupleFloat(got, want *Object) bool { if toTupleUnsafe(got).Len() != toTupleUnsafe(want).Len() { return false } for i := 0; i < toTupleUnsafe(got).Len(); i++ { if math.IsNaN(toFloatUnsafe(toTupleUnsafe(got).GetItem(i)).Value()) && math.IsNaN(toFloatUnsafe(toTupleUnsafe(want).GetItem(i)).Value()) { return true } } return false } func TestFloatCompare(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(1.0, 1.0), want: compareAllResultEq}, {args: wrapArgs(30968.3958, 30968.3958), want: compareAllResultEq}, {args: wrapArgs(-306.5, 101.0), want: compareAllResultLT}, {args: wrapArgs(309683.958, 102.1), want: compareAllResultGT}, {args: wrapArgs(0.9, 1), want: compareAllResultLT}, {args: wrapArgs(0.0, 0), want: compareAllResultEq}, {args: wrapArgs(1, 0.9), want: compareAllResultGT}, {args: wrapArgs(0, 0.0), want: compareAllResultEq}, {args: wrapArgs(0.0, None), want: compareAllResultGT}, {args: wrapArgs(math.Inf(+1), bigLongNumber), want: compareAllResultGT}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestFloatInt(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(IntType, -1209539058.2), want: NewInt(-1209539058).ToObject()}, {args: wrapArgs(IntType, 2.994514758031654e+186), want: NewLong(func() *big.Int { i, _ := big.NewFloat(2.994514758031654e+186).Int(nil); return i }()).ToObject()}, {args: wrapArgs(IntType, math.Inf(1)), wantExc: mustCreateException(OverflowErrorType, "cannot convert float infinity to integer")}, {args: wrapArgs(IntType, math.Inf(-1)), wantExc: mustCreateException(OverflowErrorType, "cannot convert float infinity to integer")}, {args: wrapArgs(IntType, math.NaN()), wantExc: mustCreateException(OverflowErrorType, "cannot convert float NaN to integer")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(IntType, "__new__", &cas); err != "" { t.Error(err) } } } func TestFloatLong(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(LongType, -3209539058.2), want: NewLong(big.NewInt(-3209539058)).ToObject()}, {args: wrapArgs(LongType, 2.994514758031654e+186), want: NewLong(func() *big.Int { i, _ := big.NewFloat(2.994514758031654e+186).Int(nil); return i }()).ToObject()}, {args: wrapArgs(LongType, math.Inf(1)), wantExc: mustCreateException(OverflowErrorType, "cannot convert float infinity to integer")}, {args: wrapArgs(LongType, math.Inf(-1)), wantExc: mustCreateException(OverflowErrorType, "cannot convert float infinity to integer")}, {args: wrapArgs(LongType, math.NaN()), wantExc: mustCreateException(OverflowErrorType, "cannot convert float NaN to integer")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(LongType, "__new__", &cas); err != "" { t.Error(err) } } } func TestFloatHash(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewFloat(0.0)), want: NewInt(0).ToObject()}, {args: wrapArgs(NewFloat(3.14)), want: NewInt(3146129223).ToObject()}, {args: wrapArgs(NewFloat(42.0)), want: NewInt(42).ToObject()}, {args: wrapArgs(NewFloat(42.125)), want: NewInt(1413677056).ToObject()}, {args: wrapArgs(NewFloat(math.Inf(1))), want: NewInt(314159).ToObject()}, {args: wrapArgs(NewFloat(math.Inf(-1))), want: NewInt(-271828).ToObject()}, {args: wrapArgs(NewFloat(math.NaN())), want: NewInt(0).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(floatHash), &cas); err != "" { t.Error(err) } } } func TestFloatIsTrue(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(0.0), want: False.ToObject()}, {args: wrapArgs(0.0001), want: True.ToObject()}, {args: wrapArgs(36983.91283), want: True.ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(IsTrue), &cas); err != "" { t.Error(err) } } } func TestFloatNew(t *testing.T) { floatNew := mustNotRaise(GetAttr(NewRootFrame(), FloatType.ToObject(), NewStr("__new__"), nil)) strictEqType := newTestClassStrictEq("StrictEq", FloatType) newStrictEq := func(v float64) *Object { f := Float{Object: Object{typ: strictEqType}, value: v} return f.ToObject() } subType := newTestClass("SubType", []*Type{FloatType}, newStringDict(map[string]*Object{})) subTypeObject := (&Float{Object: Object{typ: subType}, value: 3.14}).ToObject() goodSlotType := newTestClass("GoodSlot", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__float__": newBuiltinFunction("__float__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewFloat(3.14).ToObject(), nil }).ToObject(), })) badSlotType := newTestClass("BadSlot", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__float__": newBuiltinFunction("__float__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return newObject(ObjectType), nil }).ToObject(), })) slotSubTypeType := newTestClass("SlotSubType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__float__": newBuiltinFunction("__float__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return subTypeObject, nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(FloatType), want: NewFloat(0).ToObject()}, {args: wrapArgs(FloatType, 10.5), want: NewFloat(10.5).ToObject()}, {args: wrapArgs(FloatType, -102.1), want: NewFloat(-102.1).ToObject()}, {args: wrapArgs(FloatType, 42), want: NewFloat(42).ToObject()}, {args: wrapArgs(FloatType, "1.024e3"), want: NewFloat(1024).ToObject()}, {args: wrapArgs(FloatType, "-42"), want: NewFloat(-42).ToObject()}, {args: wrapArgs(FloatType, math.Inf(1)), want: NewFloat(math.Inf(1)).ToObject()}, {args: wrapArgs(FloatType, math.Inf(-1)), want: NewFloat(math.Inf(-1)).ToObject()}, {args: wrapArgs(FloatType, math.NaN()), want: NewFloat(math.NaN()).ToObject()}, {args: wrapArgs(FloatType, biggestFloat), want: NewFloat(math.MaxFloat64).ToObject()}, {args: wrapArgs(FloatType, new(big.Int).Neg(biggestFloat)), want: NewFloat(-math.MaxFloat64).ToObject()}, {args: wrapArgs(FloatType, new(big.Int).Sub(big.NewInt(-1), biggestFloat)), wantExc: mustCreateException(OverflowErrorType, "long int too large to convert to float")}, {args: wrapArgs(FloatType, new(big.Int).Add(biggestFloat, big.NewInt(1))), wantExc: mustCreateException(OverflowErrorType, "long int too large to convert to float")}, {args: wrapArgs(FloatType, bigLongNumber), wantExc: mustCreateException(OverflowErrorType, "long int too large to convert to float")}, {args: wrapArgs(FloatType, newObject(goodSlotType)), want: NewFloat(3.14).ToObject()}, {args: wrapArgs(FloatType, newObject(badSlotType)), wantExc: mustCreateException(TypeErrorType, "__float__ returned non-float (type object)")}, {args: wrapArgs(FloatType, newObject(slotSubTypeType)), want: subTypeObject}, {args: wrapArgs(strictEqType, 3.14), want: newStrictEq(3.14)}, {args: wrapArgs(strictEqType, newObject(goodSlotType)), want: newStrictEq(3.14)}, {args: wrapArgs(strictEqType, newObject(badSlotType)), wantExc: mustCreateException(TypeErrorType, "__float__ returned non-float (type object)")}, {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'__new__' requires 1 arguments")}, {args: wrapArgs(IntType), wantExc: mustCreateException(TypeErrorType, "float.__new__(int): int is not a subtype of float")}, {args: wrapArgs(FloatType, 123, None), wantExc: mustCreateException(TypeErrorType, "'__new__' of 'float' requires 0 or 1 arguments")}, {args: wrapArgs(FloatType, "foo"), wantExc: mustCreateException(ValueErrorType, "could not convert string to float: foo")}, {args: wrapArgs(FloatType, None), wantExc: mustCreateException(TypeErrorType, "float() argument must be a string or a number")}, } for _, cas := range cases { switch got, match := checkInvokeResult(floatNew, cas.args, cas.want, cas.wantExc); match { case checkInvokeResultExceptionMismatch: t.Errorf("float.__new__%v raised %v, want %v", cas.args, got, cas.wantExc) case checkInvokeResultReturnValueMismatch: // Handle NaN specially, since NaN != NaN. if got == nil || cas.want == nil || !got.isInstance(FloatType) || !cas.want.isInstance(FloatType) || !math.IsNaN(toFloatUnsafe(got).Value()) || !math.IsNaN(toFloatUnsafe(cas.want).Value()) { t.Errorf("float.__new__%v = %v, want %v", cas.args, got, cas.want) } } } } func TestFloatRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(0.0), want: NewStr("0.0").ToObject()}, {args: wrapArgs(0.1), want: NewStr("0.1").ToObject()}, {args: wrapArgs(-303.5), want: NewStr("-303.5").ToObject()}, {args: wrapArgs(231095835.0), want: NewStr("231095835.0").ToObject()}, {args: wrapArgs(1e+6), want: NewStr("1000000.0").ToObject()}, {args: wrapArgs(1e+15), want: NewStr("1000000000000000.0").ToObject()}, {args: wrapArgs(1e+16), want: NewStr("1e+16").ToObject()}, {args: wrapArgs(1E16), want: NewStr("1e+16").ToObject()}, {args: wrapArgs(1e-6), want: NewStr("1e-06").ToObject()}, {args: wrapArgs(math.Inf(1)), want: NewStr("inf").ToObject()}, {args: wrapArgs(math.Inf(-1)), want: NewStr("-inf").ToObject()}, {args: wrapArgs(math.NaN()), want: NewStr("nan").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func TestFloatStr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(1.0), want: NewStr("1.0").ToObject()}, {args: wrapArgs(-847.373), want: NewStr("-847.373").ToObject()}, {args: wrapArgs(0.123456789123456789), want: NewStr("0.123456789123").ToObject()}, {args: wrapArgs(1e+11), want: NewStr("100000000000.0").ToObject()}, {args: wrapArgs(1e+12), want: NewStr("1e+12").ToObject()}, {args: wrapArgs(1e-4), want: NewStr("0.0001").ToObject()}, {args: wrapArgs(1e-5), want: NewStr("1e-05").ToObject()}, {args: wrapArgs(math.Inf(1)), want: NewStr("inf").ToObject()}, {args: wrapArgs(math.Inf(-1)), want: NewStr("-inf").ToObject()}, {args: wrapArgs(math.NaN()), want: NewStr("nan").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(floatStr), &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/frame.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" ) // RunState represents the current point of execution within a Python function. type RunState int const ( notBaseExceptionMsg = "exceptions must be derived from BaseException, not %q" ) // Frame represents Python 'frame' objects. type Frame struct { Object *threadState back *Frame `attr:"f_back"` // checkpoints holds RunState values that should be executed when // unwinding the stack due to an exception. Examples of checkpoints // include exception handlers and finally blocks. checkpoints []RunState state RunState globals *Dict `attr:"f_globals"` lineno int `attr:"f_lineno"` code *Code `attr:"f_code"` taken bool } // NewRootFrame creates a Frame that is the bottom of a new stack. func NewRootFrame() *Frame { f := &Frame{Object: Object{typ: FrameType}} f.pushFrame(nil) return f } // newChildFrame creates a new Frame whose parent frame is back. func newChildFrame(back *Frame) *Frame { f := back.frameCache if f == nil { f = &Frame{Object: Object{typ: FrameType}} } else { back.frameCache, f.back = f.back, nil // Reset local state late. f.checkpoints = f.checkpoints[:0] f.state = 0 f.lineno = 0 } f.pushFrame(back) return f } func (f *Frame) release() { if !f.taken { // TODO: Track cache depth and release memory. f.frameCache, f.back = f, f.frameCache // Clear pointers early. f.setDict(nil) f.globals = nil f.code = nil } else if f.back != nil { f.back.taken = true } } // pushFrame adds f to the top of the stack, above back. func (f *Frame) pushFrame(back *Frame) { f.back = back if back == nil { f.threadState = newThreadState() } else { f.threadState = back.threadState } } func toFrameUnsafe(o *Object) *Frame { return (*Frame)(o.toPointer()) } // Globals returns the globals dict for this frame. func (f *Frame) Globals() *Dict { return f.globals } // ToObject upcasts f to an Object. func (f *Frame) ToObject() *Object { return &f.Object } // SetLineno sets the current line number for the frame. func (f *Frame) SetLineno(lineno int) { f.lineno = lineno } // State returns the current run state for f. func (f *Frame) State() RunState { return f.state } // PushCheckpoint appends state to the end of f's checkpoint stack. func (f *Frame) PushCheckpoint(state RunState) { f.checkpoints = append(f.checkpoints, state) } // PopCheckpoint removes the last element of f's checkpoint stack and returns // it. func (f *Frame) PopCheckpoint() { numCheckpoints := len(f.checkpoints) if numCheckpoints == 0 { f.state = -1 } else { f.state = f.checkpoints[numCheckpoints-1] f.checkpoints = f.checkpoints[:numCheckpoints-1] } } // Raise creates an exception and sets the exc info indicator in a way that is // compatible with the Python raise statement. The semantics are non-trivial // and are best described here: // https://docs.python.org/2/reference/simple_stmts.html#the-raise-statement // If typ, inst and tb are all nil then the currently active exception and // traceback according to ExcInfo will be used. Raise returns the exception to // propagate. func (f *Frame) Raise(typ *Object, inst *Object, tb *Object) *BaseException { if typ == nil && inst == nil && tb == nil { exc, excTraceback := f.ExcInfo() if exc != nil { typ = exc.ToObject() } if excTraceback != nil { tb = excTraceback.ToObject() } } if typ == nil { typ = None } if inst == nil { inst = None } if tb == nil { tb = None } // Build the exception if necessary. if typ.isInstance(TypeType) { t := toTypeUnsafe(typ) if !t.isSubclass(BaseExceptionType) { return f.RaiseType(TypeErrorType, fmt.Sprintf(notBaseExceptionMsg, t.Name())) } if !inst.isInstance(t) { var args Args if inst.isInstance(TupleType) { args = toTupleUnsafe(inst).elems } else if inst != None { args = []*Object{inst} } var raised *BaseException if inst, raised = typ.Call(f, args, nil); raised != nil { return raised } } } else if inst == None { inst = typ } else { return f.RaiseType(TypeErrorType, "instance exception may not have a separate value") } // Validate the exception and traceback object and raise them. if !inst.isInstance(BaseExceptionType) { return f.RaiseType(TypeErrorType, fmt.Sprintf(notBaseExceptionMsg, inst.typ.Name())) } e := toBaseExceptionUnsafe(inst) var traceback *Traceback if tb == None { traceback = newTraceback(f, nil) } else if tb.isInstance(TracebackType) { traceback = toTracebackUnsafe(tb) } else { return f.RaiseType(TypeErrorType, "raise: arg 3 must be a traceback or None") } f.RestoreExc(e, traceback) return e } // RaiseType constructs a new object of type t, passing a single str argument // built from msg and throws the constructed object. func (f *Frame) RaiseType(t *Type, msg string) *BaseException { return f.Raise(t.ToObject(), NewStr(msg).ToObject(), nil) } // ExcInfo returns the exception currently being handled by f's thread and the // associated traceback. func (f *Frame) ExcInfo() (*BaseException, *Traceback) { return f.threadState.excValue, f.threadState.excTraceback } // RestoreExc assigns the exception currently being handled by f's thread and // the associated traceback. The previously set values are returned. func (f *Frame) RestoreExc(e *BaseException, tb *Traceback) (*BaseException, *Traceback) { f.threadState.excValue, e = e, f.threadState.excValue f.threadState.excTraceback, tb = tb, f.threadState.excTraceback return e, tb } func (f *Frame) reprEnter(o *Object) bool { if f.threadState.reprState[o] { return true } if f.threadState.reprState == nil { f.threadState.reprState = map[*Object]bool{} } f.threadState.reprState[o] = true return false } func (f *Frame) reprLeave(o *Object) { delete(f.threadState.reprState, o) } // MakeArgs returns an Args slice with the given length. The slice may have // been previously used, but all elements will be set to nil. func (f *Frame) MakeArgs(n int) Args { if n == 0 { return nil } if n > argsCacheArgc { return make(Args, n) } numEntries := len(f.threadState.argsCache) if numEntries == 0 { return make(Args, n, argsCacheArgc) } args := f.threadState.argsCache[numEntries-1] f.threadState.argsCache = f.threadState.argsCache[:numEntries-1] return args[:n] } // FreeArgs clears the elements of args and returns it to the system. It may // later be returned by calls to MakeArgs and therefore references to slices of // args should not be held. func (f *Frame) FreeArgs(args Args) { if cap(args) < argsCacheArgc { return } numEntries := len(f.threadState.argsCache) if numEntries >= argsCacheSize { return } // Clear args so we don't unnecessarily hold references. for i := len(args) - 1; i >= 0; i-- { args[i] = nil } f.threadState.argsCache = f.threadState.argsCache[:numEntries+1] f.threadState.argsCache[numEntries] = args } // FrameType is the object representing the Python 'frame' type. var FrameType = newBasisType("frame", reflect.TypeOf(Frame{}), toFrameUnsafe, ObjectType) func frameExcClear(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__exc_clear__", args, FrameType); raised != nil { return nil, raised } toFrameUnsafe(args[0]).RestoreExc(nil, nil) return None, nil } func frameExcInfo(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodVarArgs(f, "__exc_info__", args, FrameType); raised != nil { return nil, raised } excObj, tbObj := None, None e, tb := toFrameUnsafe(args[0]).ExcInfo() if e != nil { excObj = e.ToObject() } if tb != nil { tbObj = tb.ToObject() } return NewTuple2(excObj, tbObj).ToObject(), nil } func initFrameType(dict map[string]*Object) { FrameType.flags &= ^(typeFlagInstantiable | typeFlagBasetype) dict["__exc_clear__"] = newBuiltinFunction("__exc_clear__", frameExcClear).ToObject() dict["__exc_info__"] = newBuiltinFunction("__exc_info__", frameExcInfo).ToObject() } ================================================ FILE: runtime/frame_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" "testing" ) const ( testRunStateInvalid RunState = -1 testRunStateStart = 0 testRunStateDone = 1 ) func TestFrameArgsCache(t *testing.T) { f := NewRootFrame() args1 := f.MakeArgs(0) if args1 != nil { t.Errorf("f.MakeArgs(0) = %v, want nil", args1) } args2 := f.MakeArgs(1) if argc := len(args2); argc != 1 { t.Errorf("f.MakeArgs(1) had len %d, want len 1", argc) } if arg0 := args2[0]; arg0 != nil { t.Errorf("f.MakeArgs(1)[0] = %v, want nil", arg0) } args2[0] = None // Make sure this is cleared in MakeArgs result below. f.FreeArgs(args2) args3 := f.MakeArgs(1) if &args2[0] != &args3[0] { t.Error("freed arg slice not returned from cache") } if arg0 := args3[0]; arg0 != nil { t.Errorf("f.MakeArgs(1)[0] = %v, want nil", arg0) } args4 := f.MakeArgs(1000) if argc := len(args4); argc != 1000 { t.Errorf("f.MakeArgs(1000) had len %d, want len 1", argc) } // Make sure the cache doesn't overflow when overfed. for i := 0; i < 100; i++ { f.FreeArgs(make(Args, argsCacheArgc)) } args5 := f.MakeArgs(2) if argc := len(args5); argc != 2 { t.Errorf("f.MakeArgs(2) had len %d, want len 2", argc) } } func TestFramePopCheckpoint(t *testing.T) { cases := []struct { states []RunState want RunState wantTop RunState }{ {nil, testRunStateInvalid, testRunStateInvalid}, {[]RunState{testRunStateDone}, testRunStateDone, testRunStateInvalid}, {[]RunState{testRunStateDone, testRunStateStart}, testRunStateStart, testRunStateDone}, } for _, cas := range cases { f := NewRootFrame() for _, state := range cas.states { f.PushCheckpoint(state) } f.PopCheckpoint() if got := f.State(); got != cas.want { t.Errorf("%#v.Pop() = %v, want %v", f, got, cas.want) } else if numCheckpoints := len(f.checkpoints); numCheckpoints == 0 && cas.wantTop != testRunStateInvalid { t.Errorf("%#v.Pop() left checkpoint stack empty, wanted top to be %v", f, cas.wantTop) } else if numCheckpoints != 0 && f.checkpoints[numCheckpoints-1] != cas.wantTop { t.Errorf("%#v.Pop() left checkpoint stack with top %v, want %v", f, f.State(), cas.wantTop) } } } func TestFramePushCheckpoint(t *testing.T) { f := NewRootFrame() states := []RunState{testRunStateStart, testRunStateDone} for _, state := range states { f.PushCheckpoint(state) if numCheckpoints := len(f.checkpoints); numCheckpoints == 0 { t.Errorf("%#v.Push(%v) left checkpoint stack empty, want non-empty", f, state) } else if top := f.checkpoints[numCheckpoints-1]; top != state { t.Errorf("%#v.Push(%v) left checkpoint stack top %v, want %v", f, state, top, state) } } } func TestFrameRaise(t *testing.T) { f := NewRootFrame() raisedFrame := NewRootFrame() raisedFrame.RestoreExc(mustCreateException(ValueErrorType, "foo"), newTraceback(raisedFrame, nil)) tb := newTraceback(f, nil) multiArgExc := toBaseExceptionUnsafe(mustNotRaise(ExceptionType.Call(f, []*Object{None, None}, nil))) barType := newTestClass("Bar", []*Type{ExceptionType}, newStringDict(map[string]*Object{ "__new__": newBuiltinFunction("__new__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("Bar").ToObject(), nil }).ToObject(), })) otherTB := newTraceback(NewRootFrame(), nil) cases := []struct { f *Frame typ *Object inst *Object tb *Object wantExc *BaseException wantTB *Traceback }{ {f, nil, nil, nil, mustCreateException(TypeErrorType, `exceptions must be derived from BaseException, not "NoneType"`), tb}, {raisedFrame, nil, nil, nil, mustCreateException(ValueErrorType, "foo"), newTraceback(raisedFrame, nil)}, {f, ExceptionType.ToObject(), nil, nil, mustCreateException(ExceptionType, ""), tb}, {f, newObject(ExceptionType), nil, nil, toBaseExceptionUnsafe(newObject(ExceptionType)), tb}, {f, NewInt(42).ToObject(), nil, nil, mustCreateException(TypeErrorType, `exceptions must be derived from BaseException, not "int"`), tb}, {f, ObjectType.ToObject(), nil, nil, mustCreateException(TypeErrorType, `exceptions must be derived from BaseException, not "object"`), tb}, {f, AssertionErrorType.ToObject(), NewStr("foo").ToObject(), nil, mustCreateException(AssertionErrorType, "foo"), tb}, {f, ExceptionType.ToObject(), NewTuple(None, None).ToObject(), nil, multiArgExc, tb}, {f, ExceptionType.ToObject(), mustCreateException(KeyErrorType, "foo").ToObject(), nil, mustCreateException(KeyErrorType, "foo"), tb}, {f, barType.ToObject(), nil, nil, mustCreateException(TypeErrorType, `exceptions must be derived from BaseException, not "str"`), tb}, {f, newObject(StopIterationType), NewInt(123).ToObject(), nil, mustCreateException(TypeErrorType, "instance exception may not have a separate value"), tb}, {f, RuntimeErrorType.ToObject(), nil, otherTB.ToObject(), mustCreateException(RuntimeErrorType, ""), otherTB}, {f, RuntimeErrorType.ToObject(), nil, newObject(ObjectType), mustCreateException(TypeErrorType, "raise: arg 3 must be a traceback or None"), tb}, } for _, cas := range cases { call := fmt.Sprintf("frame.Raise(%v, %v, %v)", cas.typ, cas.inst, cas.tb) // Not using cas.f here because the test may require // cas.f is uncleared. If a fresh frame is desired for // a particular test, use f. f.RestoreExc(nil, nil) cas.f.Raise(cas.typ, cas.inst, cas.tb) if got := cas.f.Raise(cas.typ, cas.inst, cas.tb); !exceptionsAreEquivalent(got, cas.wantExc) { t.Errorf("%s raised %v, want %v", call, got, cas.wantExc) } else if e, gotTB := cas.f.ExcInfo(); got != e { t.Errorf("%s raised %v but ExcInfo returned %v", call, got, e) } else if !reflect.DeepEqual(gotTB, cas.wantTB) { t.Errorf("%s produced traceback %v, want %v", call, gotTB, cas.wantTB) } } } func TestFrameRaiseType(t *testing.T) { fun := newBuiltinFunction("TestFrameRaiseType", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestFrameRaiseType", args, TypeType, StrType); raised != nil { return nil, raised } return nil, f.RaiseType(toTypeUnsafe(args[0]), toStrUnsafe(args[1]).Value()) }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(TypeErrorType, "bar"), wantExc: mustCreateException(TypeErrorType, "bar")}, {args: wrapArgs(ExceptionType, ""), wantExc: toBaseExceptionUnsafe(mustNotRaise(ExceptionType.Call(NewRootFrame(), wrapArgs(""), nil)))}, {args: wrapArgs(TupleType, "foo"), wantExc: mustCreateException(TypeErrorType, `exceptions must be derived from BaseException, not "tuple"`)}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestReprEnterLeave(t *testing.T) { o := newObject(ObjectType) parent := NewRootFrame() child := newChildFrame(parent) wantParent := NewRootFrame() wantParent.reprState = map[*Object]bool{o: true} child.reprEnter(o) // After child.reprEnter(), expect the parent's reprState to contain o. if wantChild := newChildFrame(parent); !reflect.DeepEqual(child, wantChild) { t.Errorf("reprEnter: child frame was %#v, want %#v", child, wantChild) } else if !reflect.DeepEqual(parent, wantParent) { t.Errorf("reprEnter: parent frame was %#v, want %#v", parent, wantParent) } else { wantParent.reprState = map[*Object]bool{} child.reprLeave(o) // Expect the parent's reprState to be empty after reprLeave(). if wantChild := newChildFrame(parent); !reflect.DeepEqual(child, wantChild) { t.Errorf("reprLeave: child frame was %#v, want %#v", child, wantChild) } else if !reflect.DeepEqual(parent, wantParent) { t.Errorf("reprLeave: parent frame was %#v, want %#v", parent, wantParent) } } } func TestFrameRoot(t *testing.T) { f1 := NewRootFrame() f2 := newChildFrame(f1) frames := []*Frame{f1, f2, newChildFrame(f2)} for _, f := range frames { if f.threadState != f1.threadState { t.Errorf("frame threadState was %v, want %v", f.threadState, f1.threadState) } } } func TestFrameExcInfo(t *testing.T) { raisedFrame := NewRootFrame() raisedExc := mustCreateException(ValueErrorType, "foo") raisedTB := newTraceback(raisedFrame, nil) raisedFrame.RestoreExc(raisedExc, raisedTB) cases := []invokeTestCase{ {args: wrapArgs(NewRootFrame()), want: NewTuple(None, None).ToObject()}, {args: wrapArgs(raisedFrame), want: newTestTuple(raisedExc, raisedTB).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(FrameType, "__exc_info__", &cas); err != "" { t.Error(err) } } } type checkInvokeResultType int const ( checkInvokeResultOk checkInvokeResultType = iota checkInvokeResultExceptionMismatch = iota checkInvokeResultReturnValueMismatch = iota ) func checkResult(got, want *Object, gotExc, wantExc *BaseException) checkInvokeResultType { if !exceptionsAreEquivalent(gotExc, wantExc) { return checkInvokeResultExceptionMismatch } if got == nil && want == nil { return checkInvokeResultOk } if got != nil && want != nil { // Compare exceptions for equivalence but other objects using // __eq__. if got.isInstance(BaseExceptionType) && want.isInstance(BaseExceptionType) && exceptionsAreEquivalent(toBaseExceptionUnsafe(got), toBaseExceptionUnsafe(want)) { return checkInvokeResultOk } f := NewRootFrame() eq, raised := Eq(f, got, want) if raised != nil { panic(raised) } b, raised := IsTrue(f, eq) if raised != nil { panic(raised) } if b { return checkInvokeResultOk } } return checkInvokeResultReturnValueMismatch } func checkInvokeResult(callable *Object, args Args, wantRet *Object, wantExc *BaseException) (*Object, checkInvokeResultType) { return checkInvokeResultKwargs(callable, args, nil, wantRet, wantExc) } func checkInvokeResultKwargs(callable *Object, args Args, kwargs KWArgs, wantRet *Object, wantExc *BaseException) (*Object, checkInvokeResultType) { ret, raised := callable.Call(NewRootFrame(), args, kwargs) switch checkResult(ret, wantRet, raised, wantExc) { case checkInvokeResultExceptionMismatch: if raised == nil { return nil, checkInvokeResultExceptionMismatch } return raised.ToObject(), checkInvokeResultExceptionMismatch case checkInvokeResultReturnValueMismatch: return ret, checkInvokeResultReturnValueMismatch default: return nil, checkInvokeResultOk } } type invokeTestCase struct { args Args kwargs KWArgs want *Object wantExc *BaseException } func runInvokeTestCase(callable *Object, cas *invokeTestCase) string { f := NewRootFrame() name := mustNotRaise(GetAttr(f, callable, internedName, NewStr("").ToObject())) if !name.isInstance(StrType) { return fmt.Sprintf("%v.__name__ is not a string", callable) } // Get repr of args before the call in case any of the args are mutated. argsRepr, raised := Repr(f, NewTuple(cas.args...).ToObject()) if raised != nil { panic(raised) } nameStr := toStrUnsafe(name).Value() switch got, match := checkInvokeResultKwargs(callable, cas.args, cas.kwargs, cas.want, cas.wantExc); match { case checkInvokeResultExceptionMismatch: return fmt.Sprintf("%s%s raised %v, want %v", nameStr, argsRepr.Value(), got, cas.wantExc) case checkInvokeResultReturnValueMismatch: return fmt.Sprintf("%s%s = %v, want %v", nameStr, argsRepr.Value(), got, cas.want) default: return "" } } func runInvokeMethodTestCase(t *Type, methodName string, cas *invokeTestCase) string { method := mustNotRaise(GetAttr(NewRootFrame(), t.ToObject(), NewStr(methodName), nil)) return runInvokeTestCase(method, cas) } ================================================ FILE: runtime/function.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" ) var ( // FunctionType is the object representing the Python 'function' type. FunctionType = newBasisType("function", reflect.TypeOf(Function{}), toFunctionUnsafe, ObjectType) // StaticMethodType is the object representing the Python // 'staticmethod' type. StaticMethodType = newBasisType("staticmethod", reflect.TypeOf(staticMethod{}), toStaticMethodUnsafe, ObjectType) // ClassMethodType is the object representing the Python // 'classmethod' type. ClassMethodType = newBasisType("classmethod", reflect.TypeOf(classMethod{}), toClassMethodUnsafe, ObjectType) ) // Args represent positional parameters in a call to a Python function. type Args []*Object func (a Args) makeCopy() Args { result := make(Args, len(a)) copy(result, a) return result } // KWArg represents a keyword argument in a call to a Python function. type KWArg struct { Name string Value *Object } // KWArgs represents a list of keyword parameters in a call to a Python // function. type KWArgs []KWArg // String returns a string representation of k, e.g. for debugging. func (k KWArgs) String() string { return k.makeDict().String() } func (k KWArgs) get(name string, def *Object) *Object { for _, kwarg := range k { if kwarg.Name == name { return kwarg.Value } } return def } func (k KWArgs) makeDict() *Dict { m := map[string]*Object{} for _, kw := range k { m[kw.Name] = kw.Value } return newStringDict(m) } // Func is a Go function underlying a Python Function object. type Func func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) // Function represents Python 'function' objects. type Function struct { Object fn Func name string `attr:"__name__"` code *Code `attr:"func_code"` globals *Dict `attr:"func_globals"` } // NewFunction creates a function object corresponding to a Python function // taking the given args, vararg and kwarg. When called, the arguments are // validated before calling fn. This includes checking that an appropriate // number of arguments are provided, populating *args and **kwargs if // necessary, etc. func NewFunction(c *Code, globals *Dict) *Function { return &Function{Object{typ: FunctionType, dict: NewDict()}, nil, c.name, c, globals} } // newBuiltinFunction returns a function object with the given name that // invokes fn when called. func newBuiltinFunction(name string, fn Func) *Function { return &Function{Object: Object{typ: FunctionType, dict: NewDict()}, fn: fn, name: name} } func toFunctionUnsafe(o *Object) *Function { return (*Function)(o.toPointer()) } // ToObject upcasts f to an Object. func (f *Function) ToObject() *Object { return &f.Object } // Name returns f's name field. func (f *Function) Name() string { return f.name } func functionCall(f *Frame, callable *Object, args Args, kwargs KWArgs) (*Object, *BaseException) { fun := toFunctionUnsafe(callable) code := fun.code if code == nil { return fun.fn(f, args, kwargs) } return code.Eval(f, fun.globals, args, kwargs) } func functionGet(f *Frame, desc, instance *Object, owner *Type) (*Object, *BaseException) { args := f.MakeArgs(3) args[0] = desc args[1] = instance args[2] = owner.ToObject() ret, raised := MethodType.Call(f, args, nil) f.FreeArgs(args) return ret, raised } func functionRepr(_ *Frame, o *Object) (*Object, *BaseException) { fun := toFunctionUnsafe(o) return NewStr(fmt.Sprintf("<%s %s at %p>", fun.typ.Name(), fun.Name(), fun)).ToObject(), nil } func initFunctionType(map[string]*Object) { FunctionType.flags &= ^(typeFlagInstantiable | typeFlagBasetype) FunctionType.slots.Call = &callSlot{functionCall} FunctionType.slots.Get = &getSlot{functionGet} FunctionType.slots.Repr = &unaryOpSlot{functionRepr} } // staticMethod represents Python 'staticmethod' objects. type staticMethod struct { Object callable *Object } func newStaticMethod(callable *Object) *staticMethod { return &staticMethod{Object{typ: StaticMethodType}, callable} } func toStaticMethodUnsafe(o *Object) *staticMethod { return (*staticMethod)(o.toPointer()) } // ToObject upcasts f to an Object. func (m *staticMethod) ToObject() *Object { return &m.Object } func staticMethodGet(f *Frame, desc, _ *Object, _ *Type) (*Object, *BaseException) { m := toStaticMethodUnsafe(desc) if m.callable == nil { return nil, f.RaiseType(RuntimeErrorType, "uninitialized staticmethod object") } return m.callable, nil } func staticMethodInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "__init__", args, ObjectType); raised != nil { return nil, raised } toStaticMethodUnsafe(o).callable = args[0] return None, nil } func initStaticMethodType(map[string]*Object) { StaticMethodType.slots.Get = &getSlot{staticMethodGet} StaticMethodType.slots.Init = &initSlot{staticMethodInit} } // classMethod represents Python 'classmethod' objects. type classMethod struct { Object callable *Object } func newClassMethod(callable *Object) *classMethod { return &classMethod{Object{typ: ClassMethodType}, callable} } func toClassMethodUnsafe(o *Object) *classMethod { return (*classMethod)(o.toPointer()) } // ToObject upcasts f to an Object. func (m *classMethod) ToObject() *Object { return &m.Object } func classMethodGet(f *Frame, desc, _ *Object, owner *Type) (*Object, *BaseException) { m := toClassMethodUnsafe(desc) if m.callable == nil { return nil, f.RaiseType(RuntimeErrorType, "uninitialized classmethod object") } args := f.MakeArgs(3) args[0] = m.callable args[1] = owner.ToObject() args[2] = args[1] ret, raised := MethodType.Call(f, args, nil) f.FreeArgs(args) return ret, raised } func classMethodInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "__init__", args, ObjectType); raised != nil { return nil, raised } toClassMethodUnsafe(o).callable = args[0] return None, nil } func initClassMethodType(map[string]*Object) { ClassMethodType.slots.Get = &getSlot{classMethodGet} ClassMethodType.slots.Init = &initSlot{classMethodInit} } ================================================ FILE: runtime/function_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "regexp" "testing" ) func TestFunctionCall(t *testing.T) { foo := newBuiltinFunction("foo", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return newTestTuple(NewTuple(args.makeCopy()...), kwargs.makeDict()).ToObject(), nil }).ToObject() bar := NewFunction(NewCode("bar", "bar.py", nil, CodeFlagVarArg, func(f *Frame, args []*Object) (*Object, *BaseException) { return args[0], nil }), nil) cases := []invokeTestCase{ {args: wrapArgs(foo, 123, "abc"), kwargs: wrapKWArgs("b", "bear"), want: newTestTuple(newTestTuple(123, "abc"), newTestDict("b", "bear")).ToObject()}, {args: wrapArgs(bar, "bar", "baz"), want: newTestTuple("bar", "baz").ToObject()}, {wantExc: mustCreateException(TypeErrorType, "unbound method __call__() must be called with function instance as first argument (got nothing instead)")}, {args: wrapArgs(newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, "unbound method __call__() must be called with function instance as first argument (got object instance instead)")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(FunctionType, "__call__", &cas); err != "" { t.Error(err) } } } func TestFunctionGet(t *testing.T) { appendMethod := mustNotRaise(GetAttr(NewRootFrame(), NewList().ToObject(), NewStr("append"), nil)) if !appendMethod.isInstance(MethodType) { t.Errorf("list.append = %v, want instancemethod", appendMethod) } } func TestFunctionName(t *testing.T) { fun := newBuiltinFunction("TestFunctionName", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { foo := newBuiltinFunction("foo", func(*Frame, Args, KWArgs) (*Object, *BaseException) { return None, nil }) return GetAttr(f, foo.ToObject(), internedName, nil) }).ToObject() if err := runInvokeTestCase(fun, &invokeTestCase{want: NewStr("foo").ToObject()}); err != "" { t.Error(err) } } func TestFunctionStrRepr(t *testing.T) { fn := func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, nil } cases := []struct { o *Object wantPattern string }{ {newBuiltinFunction("foo", fn).ToObject(), `^$`}, {newBuiltinFunction("some big function name", fn).ToObject(), `^$`}, } for _, cas := range cases { fun := wrapFuncForTest(func(f *Frame) *BaseException { re := regexp.MustCompile(cas.wantPattern) s, raised := ToStr(f, cas.o) if raised != nil { return raised } if !re.MatchString(s.Value()) { t.Errorf("str(%v) = %v, want %q", cas.o, s, re) } s, raised = Repr(f, cas.o) if raised != nil { return raised } if !re.MatchString(s.Value()) { t.Errorf("repr(%v) = %v, want %q", cas.o, s, re) } return nil }) if err := runInvokeTestCase(fun, &invokeTestCase{want: None}); err != "" { t.Error(err) } } } func TestStaticMethodGet(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newStaticMethod(NewStr("abc").ToObject()), 123, IntType), want: NewStr("abc").ToObject()}, {args: wrapArgs(newStaticMethod(nil), 123, IntType), wantExc: mustCreateException(RuntimeErrorType, "uninitialized staticmethod object")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(StaticMethodType, "__get__", &cas); err != "" { t.Error(err) } } } func TestStaticMethodInit(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Object, *BaseException) { m, raised := StaticMethodType.Call(f, args, nil) if raised != nil { return nil, raised } get, raised := GetAttr(f, m, NewStr("__get__"), nil) if raised != nil { return nil, raised } return get.Call(f, wrapArgs(123, IntType), nil) }) cases := []invokeTestCase{ {args: wrapArgs(3.14), want: NewFloat(3.14).ToObject()}, {wantExc: mustCreateException(TypeErrorType, "'__init__' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestClassMethodGet(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, meth *classMethod, args ...*Object) (*Object, *BaseException) { get, raised := GetAttr(f, meth.ToObject(), NewStr("__get__"), nil) if raised != nil { return nil, raised } callable, raised := get.Call(f, args, nil) if raised != nil { return nil, raised } return callable.Call(f, nil, nil) }) echoFunc := wrapFuncForTest(func(f *Frame, args ...*Object) *Tuple { return NewTuple(args...) }) cases := []invokeTestCase{ {args: wrapArgs(newClassMethod(echoFunc), ObjectType, ObjectType), want: NewTuple(ObjectType.ToObject()).ToObject()}, {args: wrapArgs(newClassMethod(NewStr("abc").ToObject()), 123, IntType), wantExc: mustCreateException(TypeErrorType, "first argument must be callable")}, {args: wrapArgs(newClassMethod(nil), 123, IntType), wantExc: mustCreateException(RuntimeErrorType, "uninitialized classmethod object")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestClassMethodInit(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Object, *BaseException) { m, raised := ClassMethodType.Call(f, args, nil) if raised != nil { return nil, raised } get, raised := GetAttr(f, m, NewStr("__get__"), nil) if raised != nil { return nil, raised } return get.Call(f, wrapArgs(123, IntType), nil) }) cases := []invokeTestCase{ // {args: wrapArgs(3.14), want: NewFloat(3.14).ToObject()}, {wantExc: mustCreateException(TypeErrorType, "'__init__' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/generator.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "reflect" "sync" ) var ( // GeneratorType is the object representing the Python 'generator' type. GeneratorType = newBasisType("generator", reflect.TypeOf(Generator{}), toGeneratorUnsafe, ObjectType) ) type generatorState int const ( generatorStateCreated generatorState = iota generatorStateReady generatorStateRunning generatorStateDone ) // Generator represents Python 'generator' objects. type Generator struct { Object mutex sync.Mutex state generatorState frame *Frame fn func(*Object) (*Object, *BaseException) } // NewGenerator returns a new Generator object that runs the given Block b. func NewGenerator(f *Frame, fn func(*Object) (*Object, *BaseException)) *Generator { f.taken = true // Claim the frame from being returned. // The code generator basically gives us the Frame, so we can tare it // off and prevent a parasitic `taken` from creeping up the frames. f.back = nil return &Generator{Object: Object{typ: GeneratorType}, frame: f, fn: fn} } func toGeneratorUnsafe(o *Object) *Generator { return (*Generator)(o.toPointer()) } func (g *Generator) resume(f *Frame, sendValue *Object) (*Object, *BaseException) { var raised *BaseException g.mutex.Lock() oldState := g.state switch oldState { case generatorStateCreated: if sendValue != None { raised = f.RaiseType(TypeErrorType, "can't send non-None value to a just-started generator") } else { g.state = generatorStateRunning } case generatorStateReady: g.state = generatorStateRunning case generatorStateRunning: raised = f.RaiseType(ValueErrorType, "generator already executing") case generatorStateDone: raised = f.Raise(StopIterationType.ToObject(), nil, nil) } g.mutex.Unlock() // Concurrent attempts to transition to running state will raise here // so it's guaranteed that only one thread will proceed to execute the // block below. if raised != nil { return nil, raised } g.frame.pushFrame(f) result, raised := g.fn(sendValue) g.mutex.Lock() if result == nil && raised == nil { raised = f.Raise(StopIterationType.ToObject(), nil, nil) } if raised == nil { g.frame.PopCheckpoint() g.state = generatorStateReady } else { g.state = generatorStateDone } g.mutex.Unlock() return result, raised } // ToObject upcasts g to an Object. func (g *Generator) ToObject() *Object { return &g.Object } func generatorIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func generatorNext(f *Frame, o *Object) (*Object, *BaseException) { return toGeneratorUnsafe(o).resume(f, None) } func generatorSend(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "send", args, GeneratorType, ObjectType); raised != nil { return nil, raised } return toGeneratorUnsafe(args[0]).resume(f, args[1]) } func initGeneratorType(dict map[string]*Object) { dict["send"] = newBuiltinFunction("send", generatorSend).ToObject() GeneratorType.flags &= ^(typeFlagBasetype | typeFlagInstantiable) GeneratorType.slots.Iter = &unaryOpSlot{generatorIter} GeneratorType.slots.Next = &unaryOpSlot{generatorNext} } ================================================ FILE: runtime/generator_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestGeneratorNext(t *testing.T) { f := NewRootFrame() var recursive *Object recursiveFn := func(*Object) (*Object, *BaseException) { next, raised := GetAttr(f, recursive, NewStr("next"), nil) if raised != nil { return nil, raised } return next.Call(f, nil, nil) } recursive = NewGenerator(f, recursiveFn).ToObject() emptyFn := func(*Object) (*Object, *BaseException) { return nil, nil } exhausted := NewGenerator(NewRootFrame(), emptyFn).ToObject() mustNotRaise(ListType.Call(NewRootFrame(), Args{exhausted}, nil)) cases := []invokeTestCase{ invokeTestCase{args: wrapArgs(recursive), wantExc: mustCreateException(ValueErrorType, "generator already executing")}, invokeTestCase{args: wrapArgs(exhausted), wantExc: toBaseExceptionUnsafe(mustNotRaise(StopIterationType.Call(NewRootFrame(), nil, nil)))}, } for _, cas := range cases { if err := runInvokeMethodTestCase(GeneratorType, "next", &cas); err != "" { t.Error(err) } } } func TestGeneratorSend(t *testing.T) { emptyFn := func(*Object) (*Object, *BaseException) { return nil, nil } cases := []invokeTestCase{ invokeTestCase{args: wrapArgs(NewGenerator(NewRootFrame(), emptyFn), 123), wantExc: mustCreateException(TypeErrorType, "can't send non-None value to a just-started generator")}, invokeTestCase{args: wrapArgs(NewGenerator(NewRootFrame(), emptyFn), "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "'send' of 'generator' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(GeneratorType, "send", &cas); err != "" { t.Error(err) } } } func TestGeneratorSimple(t *testing.T) { f := NewRootFrame() fn := func(*Object) (*Object, *BaseException) { switch f.State() { case 0: goto Start case 1: goto Yield1 case 2: goto Yield2 default: t.Fatalf("got invalid state %d", f.State()) } Start: f.PushCheckpoint(1) return NewStr("foo").ToObject(), nil Yield1: f.PushCheckpoint(2) return NewStr("bar").ToObject(), nil Yield2: return nil, nil } cas := &invokeTestCase{ args: wrapArgs(NewGenerator(f, fn)), want: newTestList("foo", "bar").ToObject(), } if err := runInvokeTestCase(ListType.ToObject(), cas); err != "" { t.Error(err) } } ================================================ FILE: runtime/int.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "math" "math/big" "reflect" "strconv" ) const ( internedIntMin = -2 internedIntMax = 300 ) var ( internedInts = makeInternedInts() ) // Int represents Python 'int' objects. type Int struct { Object value int } // NewInt returns a new Int holding the given integer value. func NewInt(value int) *Int { if value >= internedIntMin && value <= internedIntMax { return &internedInts[value-internedIntMin] } return &Int{Object{typ: IntType}, value} } func toIntUnsafe(o *Object) *Int { return (*Int)(o.toPointer()) } // ToObject upcasts i to an Object. func (i *Int) ToObject() *Object { return &i.Object } // Value returns the underlying integer value held by i. func (i *Int) Value() int { return i.value } // IsTrue returns false if i is zero, true otherwise. func (i *Int) IsTrue() bool { return i.Value() != 0 } // IntType is the object representing the Python 'int' type. var IntType = newBasisType("int", reflect.TypeOf(Int{}), toIntUnsafe, ObjectType) func intAbs(f *Frame, o *Object) (*Object, *BaseException) { z := toIntUnsafe(o) if z.Value() > 0 { return z.ToObject(), nil } return intNeg(f, o) } func intAdd(f *Frame, v, w *Object) (*Object, *BaseException) { return intAddMulOp(f, "__add__", v, w, intCheckedAdd, longAdd) } func intAnd(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } return NewInt(toIntUnsafe(v).Value() & toIntUnsafe(w).Value()).ToObject(), nil } func intDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return intDivModOp(f, "__div__", v, w, intCheckedDiv, longDiv) } func intDivMod(f *Frame, v, w *Object) (*Object, *BaseException) { return intDivAndModOp(f, "__divmod__", v, w, intCheckedDivMod, longDivAndMod) } func intEq(f *Frame, v, w *Object) (*Object, *BaseException) { return intCompare(compareOpEq, toIntUnsafe(v), w), nil } func intGE(f *Frame, v, w *Object) (*Object, *BaseException) { return intCompare(compareOpGE, toIntUnsafe(v), w), nil } func intGetNewArgs(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__getnewargs__", args, IntType); raised != nil { return nil, raised } return NewTuple1(args[0]).ToObject(), nil } func intGT(f *Frame, v, w *Object) (*Object, *BaseException) { return intCompare(compareOpGT, toIntUnsafe(v), w), nil } func intFloat(f *Frame, o *Object) (*Object, *BaseException) { i := toIntUnsafe(o).Value() return NewFloat(float64(i)).ToObject(), nil } func intHash(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func intHex(f *Frame, o *Object) (*Object, *BaseException) { val := numberToBase("0x", 16, o) return NewStr(val).ToObject(), nil } func intIndex(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func intInt(f *Frame, o *Object) (*Object, *BaseException) { if o.typ == IntType { return o, nil } return NewInt(toIntUnsafe(o).Value()).ToObject(), nil } func intInvert(f *Frame, o *Object) (*Object, *BaseException) { return NewInt(^toIntUnsafe(o).Value()).ToObject(), nil } func intLE(f *Frame, v, w *Object) (*Object, *BaseException) { return intCompare(compareOpLE, toIntUnsafe(v), w), nil } func intLong(f *Frame, o *Object) (*Object, *BaseException) { return NewLong(big.NewInt(int64(toIntUnsafe(o).Value()))).ToObject(), nil } func intLShift(f *Frame, v, w *Object) (*Object, *BaseException) { return intShiftOp(f, v, w, func(v, w int) (int, int, bool) { return v, w, false }) } func intLT(f *Frame, v, w *Object) (*Object, *BaseException) { return intCompare(compareOpLT, toIntUnsafe(v), w), nil } func intMod(f *Frame, v, w *Object) (*Object, *BaseException) { return intDivModOp(f, "__mod__", v, w, intCheckedMod, longMod) } func intMul(f *Frame, v, w *Object) (*Object, *BaseException) { return intAddMulOp(f, "__mul__", v, w, intCheckedMul, longMul) } func intNative(f *Frame, o *Object) (reflect.Value, *BaseException) { return reflect.ValueOf(toIntUnsafe(o).Value()), nil } func intNE(f *Frame, v, w *Object) (*Object, *BaseException) { return intCompare(compareOpNE, toIntUnsafe(v), w), nil } func intNeg(f *Frame, o *Object) (*Object, *BaseException) { z := toIntUnsafe(o) if z.Value() == MinInt { nz := big.NewInt(int64(z.Value())) return NewLong(nz.Neg(nz)).ToObject(), nil } return NewInt(-z.Value()).ToObject(), nil } func intNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { if len(args) == 0 { return newObject(t), nil } o := args[0] if len(args) == 1 && o.typ.slots.Int != nil { i, raised := ToInt(f, o) if raised != nil { return nil, raised } if t == IntType { return i, nil } n := 0 if i.isInstance(LongType) { n, raised = toLongUnsafe(i).IntValue(f) if raised != nil { return nil, raised } } else { n = toIntUnsafe(i).Value() } ret := newObject(t) toIntUnsafe(ret).value = n return ret, nil } if len(args) > 2 { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("int() takes at most 2 arguments (%d given)", len(args))) } if !o.isInstance(StrType) { if len(args) == 2 { return nil, f.RaiseType(TypeErrorType, "int() can't convert non-string with explicit base") } return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("int() argument must be a string or a number, not '%s'", o.typ.Name())) } s := toStrUnsafe(o).Value() base := 10 if len(args) == 2 { var raised *BaseException base, raised = ToIntValue(f, args[1]) if raised != nil { return nil, raised } if base < 0 || base == 1 || base > 36 { return nil, f.RaiseType(ValueErrorType, "int() base must be >= 2 and <= 36") } } i, ok := numParseInteger(new(big.Int), s, base) if !ok { format := "invalid literal for int() with base %d: %s" return nil, f.RaiseType(ValueErrorType, fmt.Sprintf(format, base, s)) } if !numInIntRange(i) { if t == IntType { return NewLong(i).ToObject(), nil } return nil, f.RaiseType(OverflowErrorType, "Python int too large to convert to a Go int") } if t != IntType { o := newObject(t) toIntUnsafe(o).value = int(i.Int64()) return o, nil } return NewInt(int(i.Int64())).ToObject(), nil } func intNonZero(f *Frame, o *Object) (*Object, *BaseException) { return GetBool(toIntUnsafe(o).Value() != 0).ToObject(), nil } func intOct(f *Frame, o *Object) (*Object, *BaseException) { val := numberToBase("0", 8, o) if val == "00" { val = "0" } return NewStr(val).ToObject(), nil } func intOr(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } return NewInt(toIntUnsafe(v).Value() | toIntUnsafe(w).Value()).ToObject(), nil } func intPos(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func intPow(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { // First try to use the faster floating point arithmetic // on the CPU, then falls back to slower methods. // IEEE float64 has 52bit of precision, so the result should be // less than MaxInt32 to be representable as an exact integer. // This assumes that int is at least 32bit. vInt := toIntUnsafe(v).Value() wInt := toIntUnsafe(w).Value() if 0 < vInt && vInt <= math.MaxInt32 && 0 < wInt && wInt <= math.MaxInt32 { res := math.Pow(float64(vInt), float64(wInt)) // Can the result be interpreted as an int? if !math.IsNaN(res) && !math.IsInf(res, 0) && res <= math.MaxInt32 { return NewInt(int(res)).ToObject(), nil } } // Special cases. if vInt == 0 { if wInt < 0 { return nil, f.RaiseType(ZeroDivisionErrorType, "0.0 cannot be raised to a negative power") } if wInt == 0 { return NewInt(1).ToObject(), nil } return NewInt(0).ToObject(), nil } // If w < 0, the result must be a floating point number. // We convert both arguments to float and continue. if wInt < 0 { return floatPow(f, NewFloat(float64(vInt)).ToObject(), NewFloat(float64(wInt)).ToObject()) } // Else we convert to Long and continue there. return longPow(f, NewLong(big.NewInt(int64(vInt))).ToObject(), NewLong(big.NewInt(int64(wInt))).ToObject()) } return NotImplemented, nil } func intRAdd(f *Frame, v, w *Object) (*Object, *BaseException) { return intAddMulOp(f, "__radd__", v, w, intCheckedAdd, longAdd) } func intRDiv(f *Frame, v, w *Object) (*Object, *BaseException) { return intDivModOp(f, "__rdiv__", v, w, func(v, w int) (int, divModResult) { return intCheckedDiv(w, v) }, func(z, x, y *big.Int) { longDiv(z, y, x) }) } func intRDivMod(f *Frame, v, w *Object) (*Object, *BaseException) { return intDivAndModOp(f, "__rdivmod__", v, w, func(v, w int) (int, int, divModResult) { return intCheckedDivMod(w, v) }, func(z, m, x, y *big.Int) { longDivAndMod(z, m, y, x) }) } func intRepr(f *Frame, o *Object) (*Object, *BaseException) { return NewStr(strconv.FormatInt(int64(toIntUnsafe(o).Value()), 10)).ToObject(), nil } func intRMod(f *Frame, v, w *Object) (*Object, *BaseException) { return intDivModOp(f, "__rmod__", v, w, func(v, w int) (int, divModResult) { return intCheckedMod(w, v) }, func(z, x, y *big.Int) { longMod(z, y, x) }) } func intRMul(f *Frame, v, w *Object) (*Object, *BaseException) { return intAddMulOp(f, "__rmul__", v, w, intCheckedMul, longMul) } func intRLShift(f *Frame, v, w *Object) (*Object, *BaseException) { return intShiftOp(f, v, w, func(v, w int) (int, int, bool) { return w, v, false }) } func intRRShift(f *Frame, v, w *Object) (*Object, *BaseException) { return intShiftOp(f, v, w, func(v, w int) (int, int, bool) { return w, v, true }) } func intRShift(f *Frame, v, w *Object) (*Object, *BaseException) { return intShiftOp(f, v, w, func(v, w int) (int, int, bool) { return v, w, true }) } func intRSub(f *Frame, v, w *Object) (*Object, *BaseException) { return intAddMulOp(f, "__rsub__", v, w, func(v, w int) (int, bool) { return intCheckedSub(w, v) }, func(z, x, y *big.Int) { longSub(z, y, x) }) } func intSub(f *Frame, v, w *Object) (*Object, *BaseException) { return intAddMulOp(f, "__sub__", v, w, intCheckedSub, longSub) } func intXor(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } return NewInt(toIntUnsafe(v).Value() ^ toIntUnsafe(w).Value()).ToObject(), nil } func initIntType(dict map[string]*Object) { dict["__getnewargs__"] = newBuiltinFunction("__getnewargs__", intGetNewArgs).ToObject() IntType.slots.Abs = &unaryOpSlot{intAbs} IntType.slots.Add = &binaryOpSlot{intAdd} IntType.slots.And = &binaryOpSlot{intAnd} IntType.slots.Div = &binaryOpSlot{intDiv} IntType.slots.DivMod = &binaryOpSlot{intDivMod} IntType.slots.Eq = &binaryOpSlot{intEq} IntType.slots.FloorDiv = &binaryOpSlot{intDiv} IntType.slots.GE = &binaryOpSlot{intGE} IntType.slots.GT = &binaryOpSlot{intGT} IntType.slots.Float = &unaryOpSlot{intFloat} IntType.slots.Hash = &unaryOpSlot{intHash} IntType.slots.Hex = &unaryOpSlot{intHex} IntType.slots.Index = &unaryOpSlot{intIndex} IntType.slots.Int = &unaryOpSlot{intInt} IntType.slots.Invert = &unaryOpSlot{intInvert} IntType.slots.LE = &binaryOpSlot{intLE} IntType.slots.LShift = &binaryOpSlot{intLShift} IntType.slots.LT = &binaryOpSlot{intLT} IntType.slots.Long = &unaryOpSlot{intLong} IntType.slots.Mod = &binaryOpSlot{intMod} IntType.slots.Mul = &binaryOpSlot{intMul} IntType.slots.Native = &nativeSlot{intNative} IntType.slots.NE = &binaryOpSlot{intNE} IntType.slots.Neg = &unaryOpSlot{intNeg} IntType.slots.New = &newSlot{intNew} IntType.slots.NonZero = &unaryOpSlot{intNonZero} IntType.slots.Oct = &unaryOpSlot{intOct} IntType.slots.Or = &binaryOpSlot{intOr} IntType.slots.Pos = &unaryOpSlot{intPos} IntType.slots.Pow = &binaryOpSlot{intPow} IntType.slots.RAdd = &binaryOpSlot{intRAdd} IntType.slots.RAnd = &binaryOpSlot{intAnd} IntType.slots.RDiv = &binaryOpSlot{intRDiv} IntType.slots.RDivMod = &binaryOpSlot{intRDivMod} IntType.slots.Repr = &unaryOpSlot{intRepr} IntType.slots.RFloorDiv = &binaryOpSlot{intRDiv} IntType.slots.RMod = &binaryOpSlot{intRMod} IntType.slots.RMul = &binaryOpSlot{intRMul} IntType.slots.ROr = &binaryOpSlot{intOr} IntType.slots.RLShift = &binaryOpSlot{intRLShift} IntType.slots.RRShift = &binaryOpSlot{intRRShift} IntType.slots.RShift = &binaryOpSlot{intRShift} IntType.slots.RSub = &binaryOpSlot{intRSub} IntType.slots.RXor = &binaryOpSlot{intXor} IntType.slots.Sub = &binaryOpSlot{intSub} IntType.slots.Xor = &binaryOpSlot{intXor} } type divModResult int const ( divModOK divModResult = iota divModOverflow = iota divModZeroDivision = iota ) func intCompare(op compareOp, v *Int, w *Object) *Object { if !w.isInstance(IntType) { return NotImplemented } lhs, rhs := v.Value(), toIntUnsafe(w).Value() result := false switch op { case compareOpLT: result = lhs < rhs case compareOpLE: result = lhs <= rhs case compareOpEq: result = lhs == rhs case compareOpNE: result = lhs != rhs case compareOpGE: result = lhs >= rhs case compareOpGT: result = lhs > rhs } return GetBool(result).ToObject() } func intAddMulOp(f *Frame, method string, v, w *Object, fun func(v, w int) (int, bool), bigFun func(z, x, y *big.Int)) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } r, ok := fun(toIntUnsafe(v).Value(), toIntUnsafe(w).Value()) if !ok { return longCallBinary(bigFun, intToLong(toIntUnsafe(v)), intToLong(toIntUnsafe(w))), nil } return NewInt(r).ToObject(), nil } func intCheckedDiv(v, w int) (int, divModResult) { q, _, r := intCheckedDivMod(v, w) return q, r } func intCheckedDivMod(v, w int) (int, int, divModResult) { if w == 0 { return 0, 0, divModZeroDivision } if v == MinInt && w == -1 { return 0, 0, divModOverflow } q := v / w m := v % w if m != 0 && (w^m) < 0 { // In Python the result of the modulo operator is always the // same sign as the divisor, whereas in Go, the result is // always the same sign as the dividend. Therefore we need to // do an adjustment when the sign of the modulo result differs // from that of the divisor. m += w // Relatedly, in Python the result of division truncates toward // negative infinity whereas it truncates toward zero in Go. // The fact that the signs of the divisor and the modulo result // differ implies that the quotient is also negative so we also // adjust the quotient here. q-- } return q, m, divModOK } func intCheckedAdd(v, w int) (int, bool) { if (v > 0 && w > MaxInt-v) || (v < 0 && w < MinInt-v) { return 0, false } return v + w, true } func intCheckedMod(v, w int) (int, divModResult) { _, m, r := intCheckedDivMod(v, w) return m, r } func intCheckedMul(v, w int) (int, bool) { if v == 0 || w == 0 || v == 1 || w == 1 { return v * w, true } // Since MinInt can only be multiplied by zero and one safely and we've // already handled that case above, we know this multiplication will // overflow. Unfortunately the division check below will fail to catch // this by coincidence: MinInt * -1 overflows to MinInt, causing the // expression x/w to overflow, coincidentally producing MinInt which // makes it seem as though the multiplication was correct. if v == MinInt || w == MinInt { return 0, false } x := v * w if x/w != v { return 0, false } return x, true } func intCheckedSub(v, w int) (int, bool) { if (w > 0 && v < MinInt+w) || (w < 0 && v > MaxInt+w) { return 0, false } return v - w, true } func intDivModOp(f *Frame, method string, v, w *Object, fun func(v, w int) (int, divModResult), bigFun func(z, x, y *big.Int)) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } x, r := fun(toIntUnsafe(v).Value(), toIntUnsafe(w).Value()) switch r { case divModOverflow: return longCallBinary(bigFun, intToLong(toIntUnsafe(v)), intToLong(toIntUnsafe(w))), nil case divModZeroDivision: return nil, f.RaiseType(ZeroDivisionErrorType, "integer division or modulo by zero") } return NewInt(x).ToObject(), nil } func intDivAndModOp(f *Frame, method string, v, w *Object, fun func(v, w int) (int, int, divModResult), bigFun func(z, m, x, y *big.Int)) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } q, m, r := fun(toIntUnsafe(v).Value(), toIntUnsafe(w).Value()) switch r { case divModOverflow: return longCallBinaryTuple(bigFun, intToLong(toIntUnsafe(v)), intToLong(toIntUnsafe(w))), nil case divModZeroDivision: return nil, f.RaiseType(ZeroDivisionErrorType, "integer division or modulo by zero") } return NewTuple2(NewInt(q).ToObject(), NewInt(m).ToObject()).ToObject(), nil } func intShiftOp(f *Frame, v, w *Object, fun func(int, int) (int, int, bool)) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } lhs, rhs, rshift := fun(toIntUnsafe(v).Value(), toIntUnsafe(w).Value()) if rhs < 0 { return nil, f.RaiseType(ValueErrorType, "negative shift count") } var result int n := uint(rhs) if rshift { result = lhs >> n } else { result = lhs << n if result>>n != lhs { return NewLong(new(big.Int).Lsh(big.NewInt(int64(lhs)), n)).ToObject(), nil } } return NewInt(result).ToObject(), nil } func intToLong(o *Int) *Long { return NewLong(big.NewInt(int64(o.Value()))) } func makeInternedInts() [internedIntMax - internedIntMin + 1]Int { var ints [internedIntMax - internedIntMin + 1]Int for i := internedIntMin; i <= internedIntMax; i++ { ints[i-internedIntMin] = Int{Object{typ: IntType}, i} } return ints } ================================================ FILE: runtime/int_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "math/big" "runtime" "testing" ) func TestIntBinaryOps(t *testing.T) { cases := []struct { fun binaryOpFunc v, w *Object want *Object wantExc *BaseException }{ {Add, NewInt(-100).ToObject(), NewInt(50).ToObject(), NewInt(-50).ToObject(), nil}, {Add, newObject(ObjectType), NewInt(-100).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'object' and 'int'")}, {Add, NewInt(MaxInt).ToObject(), NewInt(1).ToObject(), NewLong(new(big.Int).Add(maxIntBig, big.NewInt(1))).ToObject(), nil}, {And, NewInt(-100).ToObject(), NewInt(50).ToObject(), NewInt(16).ToObject(), nil}, {And, NewInt(MaxInt).ToObject(), NewInt(MinInt).ToObject(), NewInt(0).ToObject(), nil}, {And, newObject(ObjectType), NewInt(-100).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for &: 'object' and 'int'")}, {Div, NewInt(7).ToObject(), NewInt(3).ToObject(), NewInt(2).ToObject(), nil}, {Div, NewInt(MaxInt).ToObject(), NewInt(MinInt).ToObject(), NewInt(-1).ToObject(), nil}, {Div, NewInt(MinInt).ToObject(), NewInt(MaxInt).ToObject(), NewInt(-2).ToObject(), nil}, {Div, NewList().ToObject(), NewInt(21).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for /: 'list' and 'int'")}, {Div, NewInt(1).ToObject(), NewInt(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "integer division or modulo by zero")}, {Div, NewInt(MinInt).ToObject(), NewInt(-1).ToObject(), NewLong(new(big.Int).Neg(minIntBig)).ToObject(), nil}, {DivMod, NewInt(7).ToObject(), NewInt(3).ToObject(), NewTuple2(NewInt(2).ToObject(), NewInt(1).ToObject()).ToObject(), nil}, {DivMod, NewInt(3).ToObject(), NewInt(-7).ToObject(), NewTuple2(NewInt(-1).ToObject(), NewInt(-4).ToObject()).ToObject(), nil}, {DivMod, NewInt(MaxInt).ToObject(), NewInt(MinInt).ToObject(), NewTuple2(NewInt(-1).ToObject(), NewInt(-1).ToObject()).ToObject(), nil}, {DivMod, NewInt(MinInt).ToObject(), NewInt(MaxInt).ToObject(), NewTuple2(NewInt(-2).ToObject(), NewInt(MaxInt-1).ToObject()).ToObject(), nil}, {DivMod, NewList().ToObject(), NewInt(21).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for divmod(): 'list' and 'int'")}, {DivMod, NewInt(1).ToObject(), NewInt(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "integer division or modulo by zero")}, {DivMod, NewInt(MinInt).ToObject(), NewInt(-1).ToObject(), NewTuple2(NewLong(new(big.Int).Neg(minIntBig)).ToObject(), NewLong(big.NewInt(0)).ToObject()).ToObject(), nil}, {FloorDiv, NewInt(7).ToObject(), NewInt(3).ToObject(), NewInt(2).ToObject(), nil}, {FloorDiv, NewInt(MaxInt).ToObject(), NewInt(MinInt).ToObject(), NewInt(-1).ToObject(), nil}, {FloorDiv, NewInt(MinInt).ToObject(), NewInt(MaxInt).ToObject(), NewInt(-2).ToObject(), nil}, {FloorDiv, NewList().ToObject(), NewInt(21).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for //: 'list' and 'int'")}, {FloorDiv, NewInt(1).ToObject(), NewInt(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "integer division or modulo by zero")}, {FloorDiv, NewInt(MinInt).ToObject(), NewInt(-1).ToObject(), NewLong(new(big.Int).Neg(minIntBig)).ToObject(), nil}, {LShift, NewInt(2).ToObject(), NewInt(4).ToObject(), NewInt(32).ToObject(), nil}, {LShift, NewInt(-12).ToObject(), NewInt(10).ToObject(), NewInt(-12288).ToObject(), nil}, {LShift, NewInt(10).ToObject(), NewInt(100).ToObject(), NewLong(new(big.Int).Lsh(big.NewInt(10), 100)).ToObject(), nil}, {LShift, NewInt(2).ToObject(), NewInt(-5).ToObject(), nil, mustCreateException(ValueErrorType, "negative shift count")}, {LShift, NewInt(4).ToObject(), NewFloat(3.14).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for <<: 'int' and 'float'")}, {LShift, newObject(ObjectType), NewInt(4).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for <<: 'object' and 'int'")}, {RShift, NewInt(87).ToObject(), NewInt(3).ToObject(), NewInt(10).ToObject(), nil}, {RShift, NewInt(-101).ToObject(), NewInt(5).ToObject(), NewInt(-4).ToObject(), nil}, {RShift, NewInt(12).ToObject(), NewInt(10).ToObject(), NewInt(0).ToObject(), nil}, {RShift, NewInt(12).ToObject(), NewInt(-10).ToObject(), nil, mustCreateException(ValueErrorType, "negative shift count")}, {RShift, NewInt(4).ToObject(), NewFloat(3.14).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for >>: 'int' and 'float'")}, {RShift, newObject(ObjectType), NewInt(4).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for >>: 'object' and 'int'")}, {RShift, NewInt(4).ToObject(), newObject(ObjectType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for >>: 'int' and 'object'")}, {Mod, NewInt(3).ToObject(), NewInt(-7).ToObject(), NewInt(-4).ToObject(), nil}, {Mod, NewInt(MaxInt).ToObject(), NewInt(MinInt).ToObject(), NewInt(-1).ToObject(), nil}, {Mod, NewInt(MinInt).ToObject(), NewInt(MaxInt).ToObject(), NewInt(MaxInt - 1).ToObject(), nil}, {Mod, None, NewInt(-4).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for %: 'NoneType' and 'int'")}, {Mod, NewInt(10).ToObject(), NewInt(0).ToObject(), nil, mustCreateException(ZeroDivisionErrorType, "integer division or modulo by zero")}, {Mod, NewInt(MinInt).ToObject(), NewInt(-1).ToObject(), NewLong(big.NewInt(0)).ToObject(), nil}, {Mul, NewInt(-1).ToObject(), NewInt(-3).ToObject(), NewInt(3).ToObject(), nil}, {Mul, newObject(ObjectType), NewInt(101).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'object' and 'int'")}, {Mul, NewInt(MaxInt).ToObject(), NewInt(MaxInt - 1).ToObject(), NewLong(new(big.Int).Mul(big.NewInt(MaxInt), big.NewInt(MaxInt-1))).ToObject(), nil}, {Or, NewInt(-100).ToObject(), NewInt(50).ToObject(), NewInt(-66).ToObject(), nil}, {Or, NewInt(MaxInt).ToObject(), NewInt(MinInt).ToObject(), NewInt(-1).ToObject(), nil}, {Or, newObject(ObjectType), NewInt(-100).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for |: 'object' and 'int'")}, {Pow, NewInt(2).ToObject(), NewInt(128).ToObject(), NewLong(big.NewInt(0).Exp(big.NewInt(2), big.NewInt(128), nil)).ToObject(), nil}, {Pow, NewInt(2).ToObject(), newObject(ObjectType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for **: 'int' and 'object'")}, {Pow, NewInt(2).ToObject(), NewInt(-2).ToObject(), NewFloat(0.25).ToObject(), nil}, {Pow, newObject(ObjectType), NewInt(2).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for **: 'object' and 'int'")}, {Sub, NewInt(22).ToObject(), NewInt(18).ToObject(), NewInt(4).ToObject(), nil}, {Sub, IntType.ToObject(), NewInt(42).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for -: 'type' and 'int'")}, {Sub, NewInt(MinInt).ToObject(), NewInt(1).ToObject(), NewLong(new(big.Int).Sub(minIntBig, big.NewInt(1))).ToObject(), nil}, {Xor, NewInt(-100).ToObject(), NewInt(50).ToObject(), NewInt(-82).ToObject(), nil}, {Xor, NewInt(MaxInt).ToObject(), NewInt(MinInt).ToObject(), NewInt(-1).ToObject(), nil}, {Xor, newObject(ObjectType), NewInt(-100).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for ^: 'object' and 'int'")}, } for _, cas := range cases { testCase := invokeTestCase{args: wrapArgs(cas.v, cas.w), want: cas.want, wantExc: cas.wantExc} if err := runInvokeTestCase(wrapFuncForTest(cas.fun), &testCase); err != "" { t.Error(err) } } } func TestIntCompare(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(1, 1), want: compareAllResultEq}, {args: wrapArgs(309683958, 309683958), want: compareAllResultEq}, {args: wrapArgs(-306, 101), want: compareAllResultLT}, {args: wrapArgs(309683958, 101), want: compareAllResultGT}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestIntInvert(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(2592), want: NewInt(-2593).ToObject()}, {args: wrapArgs(0), want: NewInt(-1).ToObject()}, {args: wrapArgs(-43), want: NewInt(42).ToObject()}, {args: wrapArgs(MaxInt), want: NewInt(MinInt).ToObject()}, {args: wrapArgs(MinInt), want: NewInt(MaxInt).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(IntType, "__invert__", &cas); err != "" { t.Error(err) } } } func TestIntNew(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__int__": newBuiltinFunction("__int__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[0], nil }).ToObject(), })) strictEqType := newTestClassStrictEq("StrictEq", IntType) subType := newTestClass("SubType", []*Type{IntType}, newStringDict(map[string]*Object{})) subTypeObject := (&Int{Object: Object{typ: subType}, value: 3}).ToObject() goodSlotType := newTestClass("GoodSlot", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__int__": newBuiltinFunction("__int__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(3).ToObject(), nil }).ToObject(), })) badSlotType := newTestClass("BadSlot", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__int__": newBuiltinFunction("__int__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return newObject(ObjectType), nil }).ToObject(), })) slotSubTypeType := newTestClass("SlotSubType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__int__": newBuiltinFunction("__int__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return subTypeObject, nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(IntType), want: NewInt(0).ToObject()}, {args: wrapArgs(IntType, "123"), want: NewInt(123).ToObject()}, {args: wrapArgs(IntType, " \t123"), want: NewInt(123).ToObject()}, {args: wrapArgs(IntType, "123 \t"), want: NewInt(123).ToObject()}, {args: wrapArgs(IntType, "FF", 16), want: NewInt(255).ToObject()}, {args: wrapArgs(IntType, "0xFF", 16), want: NewInt(255).ToObject()}, {args: wrapArgs(IntType, "0xE", 0), want: NewInt(14).ToObject()}, {args: wrapArgs(IntType, "0b101", 0), want: NewInt(5).ToObject()}, {args: wrapArgs(IntType, "0o726", 0), want: NewInt(470).ToObject()}, {args: wrapArgs(IntType, "0726", 0), want: NewInt(470).ToObject()}, {args: wrapArgs(IntType, "102", 0), want: NewInt(102).ToObject()}, {args: wrapArgs(IntType, 42), want: NewInt(42).ToObject()}, {args: wrapArgs(IntType, -3.14), want: NewInt(-3).ToObject()}, {args: wrapArgs(subType, overflowLong), wantExc: mustCreateException(OverflowErrorType, "Python int too large to convert to a Go int")}, {args: wrapArgs(strictEqType, 42), want: (&Int{Object{typ: strictEqType}, 42}).ToObject()}, {args: wrapArgs(IntType, newObject(goodSlotType)), want: NewInt(3).ToObject()}, {args: wrapArgs(IntType, newObject(badSlotType)), wantExc: mustCreateException(TypeErrorType, "__int__ returned non-int (type object)")}, {args: wrapArgs(IntType, newObject(slotSubTypeType)), want: subTypeObject}, {args: wrapArgs(strictEqType, newObject(goodSlotType)), want: (&Int{Object{typ: strictEqType}, 3}).ToObject()}, {args: wrapArgs(strictEqType, newObject(badSlotType)), wantExc: mustCreateException(TypeErrorType, "__int__ returned non-int (type object)")}, {args: wrapArgs(IntType, "0xff"), wantExc: mustCreateException(ValueErrorType, "invalid literal for int() with base 10: 0xff")}, {args: wrapArgs(IntType, ""), wantExc: mustCreateException(ValueErrorType, "invalid literal for int() with base 10: ")}, {args: wrapArgs(IntType, " "), wantExc: mustCreateException(ValueErrorType, "invalid literal for int() with base 10: ")}, {args: wrapArgs(FloatType), wantExc: mustCreateException(TypeErrorType, "int.__new__(float): float is not a subtype of int")}, {args: wrapArgs(IntType, "asldkfj", 1), wantExc: mustCreateException(ValueErrorType, "int() base must be >= 2 and <= 36")}, {args: wrapArgs(IntType, "asldkfj", 37), wantExc: mustCreateException(ValueErrorType, "int() base must be >= 2 and <= 36")}, {args: wrapArgs(IntType, "@#%*(#", 36), wantExc: mustCreateException(ValueErrorType, "invalid literal for int() with base 36: @#%*(#")}, {args: wrapArgs(IntType, "123", overflowLong), wantExc: mustCreateException(OverflowErrorType, "Python int too large to convert to a Go int")}, {args: wrapArgs(IntType, "32059823095809238509238590835"), want: NewLong(func() *big.Int { i, _ := new(big.Int).SetString("32059823095809238509238590835", 0); return i }()).ToObject()}, {args: wrapArgs(IntType, newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, "int() argument must be a string or a number, not 'object'")}, {args: wrapArgs(IntType, newObject(fooType)), wantExc: mustCreateException(TypeErrorType, "__int__ returned non-int (type Foo)")}, {args: wrapArgs(IntType, 1, 2), wantExc: mustCreateException(TypeErrorType, "int() can't convert non-string with explicit base")}, {args: wrapArgs(IntType, 1, 2, 3), wantExc: mustCreateException(TypeErrorType, "int() takes at most 2 arguments (3 given)")}, {args: wrapArgs(IntType, "1", None), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(IntType, "__new__", &cas); err != "" { t.Error(err) } } } func TestIntNewInterned(t *testing.T) { // Make sure small integers are interned. fun := wrapFuncForTest(func(f *Frame, i *Int) (bool, *BaseException) { o, raised := IntType.Call(f, wrapArgs(i.Value()), nil) if raised != nil { return false, raised } return o == i.ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(-1001), want: False.ToObject()}, {args: wrapArgs(0), want: True.ToObject()}, {args: wrapArgs(100), want: True.ToObject()}, {args: wrapArgs(120948298), want: False.ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func BenchmarkIntNew(b *testing.B) { b.Run("interned", func(b *testing.B) { var ret *Object for i := 0; i < b.N; i++ { ret = NewInt(1).ToObject() } runtime.KeepAlive(ret) }) b.Run("not interned", func(b *testing.B) { var ret *Object for i := 0; i < b.N; i++ { ret = NewInt(internedIntMax + 5).ToObject() } runtime.KeepAlive(ret) }) } func TestIntStrRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(0), want: NewStr("0").ToObject()}, {args: wrapArgs(-303), want: NewStr("-303").ToObject()}, {args: wrapArgs(231095835), want: NewStr("231095835").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func TestIntCheckedAddMul(t *testing.T) { cases := []struct { f func(a, b int) (int, bool) a, b int want int wantOK bool }{ {intCheckedAdd, 1, 2, 3, true}, {intCheckedAdd, MaxInt, -1, MaxInt - 1, true}, {intCheckedAdd, MaxInt, 0, MaxInt, true}, {intCheckedAdd, MaxInt, 1, 0, false}, {intCheckedAdd, MinInt, -1, 0, false}, {intCheckedAdd, MinInt, 0, MinInt, true}, {intCheckedAdd, MinInt, 1, MinInt + 1, true}, {intCheckedMul, MaxInt, 1, MaxInt, true}, {intCheckedMul, MaxInt, -1, MinInt + 1, true}, {intCheckedMul, MinInt, -1, 0, false}, } for _, cas := range cases { if got, gotOK := cas.f(cas.a, cas.b); got != cas.want || gotOK != cas.wantOK { t.Errorf("%s(%v, %v) = (%v, %v), want (%v, %v)", getFuncName(cas.f), cas.a, cas.b, got, gotOK, cas.want, cas.wantOK) } if got, gotOK := cas.f(cas.b, cas.a); got != cas.want || gotOK != cas.wantOK { t.Errorf("%s(%v, %v) = (%v, %v), want (%v, %v)", getFuncName(cas.f), cas.b, cas.a, got, gotOK, cas.want, cas.wantOK) } } } func TestIntCheckedDivMod(t *testing.T) { cases := []struct { f func(a, b int) (int, divModResult) a, b int want int wantResult divModResult }{ {intCheckedDiv, 872, 736, 1, divModOK}, {intCheckedDiv, -320, 3, -107, divModOK}, {intCheckedDiv, 7, 3, 2, divModOK}, {intCheckedDiv, 7, -3, -3, divModOK}, {intCheckedDiv, -7, 3, -3, divModOK}, {intCheckedDiv, -7, -3, 2, divModOK}, {intCheckedDiv, 3, 7, 0, divModOK}, {intCheckedDiv, 3, -7, -1, divModOK}, {intCheckedDiv, -3, 7, -1, divModOK}, {intCheckedDiv, -3, -7, 0, divModOK}, {intCheckedDiv, MaxInt, MaxInt, 1, divModOK}, {intCheckedDiv, MaxInt, MinInt, -1, divModOK}, {intCheckedDiv, MinInt, MaxInt, -2, divModOK}, {intCheckedDiv, MinInt, MinInt, 1, divModOK}, {intCheckedDiv, 22, 0, 0, divModZeroDivision}, {intCheckedDiv, MinInt, -1, 0, divModOverflow}, {intCheckedMod, -142, -118, -24, divModOK}, {intCheckedMod, -225, 454, 229, divModOK}, {intCheckedMod, 7, 3, 1, divModOK}, {intCheckedMod, 7, -3, -2, divModOK}, {intCheckedMod, -7, 3, 2, divModOK}, {intCheckedMod, -7, -3, -1, divModOK}, {intCheckedMod, 3, 7, 3, divModOK}, {intCheckedMod, 3, -7, -4, divModOK}, {intCheckedMod, -3, 7, 4, divModOK}, {intCheckedMod, -3, -7, -3, divModOK}, {intCheckedMod, MaxInt, MaxInt, 0, divModOK}, {intCheckedMod, MaxInt, MinInt, -1, divModOK}, {intCheckedMod, MinInt, MaxInt, MaxInt - 1, divModOK}, {intCheckedMod, MinInt, MinInt, 0, divModOK}, {intCheckedMod, -50, 0, 0, divModZeroDivision}, {intCheckedMod, MinInt, -1, 0, divModOverflow}, } for _, cas := range cases { if got, gotResult := cas.f(cas.a, cas.b); got != cas.want || gotResult != cas.wantResult { t.Errorf("%s(%v, %v) = (%v, %v), want (%v, %v)", getFuncName(cas.f), cas.a, cas.b, got, gotResult, cas.want, cas.wantResult) } } } func TestIntCheckedSub(t *testing.T) { cases := []struct { f func(a, b int) (int, bool) a, b int want int wantOK bool }{ {intCheckedSub, MaxInt, MaxInt, 0, true}, {intCheckedSub, MaxInt, -1, 0, false}, {intCheckedSub, MaxInt, 0, MaxInt, true}, {intCheckedSub, MaxInt, 1, MaxInt - 1, true}, {intCheckedSub, MinInt, -1, MinInt + 1, true}, {intCheckedSub, MinInt, 0, MinInt, true}, {intCheckedSub, MinInt, 1, 0, false}, {intCheckedSub, MinInt, MinInt, 0, true}, {intCheckedSub, -2, MaxInt, 0, false}, {intCheckedSub, -1, MaxInt, MinInt, true}, {intCheckedSub, 0, MaxInt, MinInt + 1, true}, {intCheckedSub, -1, MinInt, MaxInt, true}, {intCheckedSub, 0, MinInt, 0, false}, } for _, cas := range cases { if got, gotOK := cas.f(cas.a, cas.b); got != cas.want || gotOK != cas.wantOK { t.Errorf("%s(%v, %v) = (%v, %v), want (%v, %v)", getFuncName(cas.f), cas.a, cas.b, got, gotOK, cas.want, cas.wantOK) } } } ================================================ FILE: runtime/list.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" "sort" "sync" ) // List represents Python 'list' objects. // // Lists are thread safe, however read operations are not necessarily atomic. // E.g. given the list l = [1, 2, 3] executing del l[1] in one thread may give // repr(l) == [1, 2] in another which is never correct. type List struct { Object mutex sync.RWMutex elems []*Object } // NewList returns a list containing the given elements. func NewList(elems ...*Object) *List { l := &List{Object: Object{typ: ListType}} numElems := len(elems) l.resize(numElems) for i := 0; i < numElems; i++ { l.elems[i] = elems[i] } return l } func toListUnsafe(o *Object) *List { return (*List)(o.toPointer()) } // ToObject upcasts l to an Object. func (l *List) ToObject() *Object { return &l.Object } // Append adds o to the end of l. func (l *List) Append(o *Object) { l.mutex.Lock() newLen := len(l.elems) + 1 l.resize(newLen) l.elems[newLen-1] = o l.mutex.Unlock() } // DelItem removes the index'th element of l. func (l *List) DelItem(f *Frame, index int) *BaseException { l.mutex.Lock() numElems := len(l.elems) i, raised := seqCheckedIndex(f, numElems, index) if raised == nil { copy(l.elems[i:numElems-1], l.elems[i+1:numElems]) l.elems = l.elems[:numElems-1] } l.mutex.Unlock() return raised } // DelSlice removes the slice of l specified by s. func (l *List) DelSlice(f *Frame, s *Slice) *BaseException { l.mutex.Lock() numListElems := len(l.elems) start, stop, step, numSliceElems, raised := s.calcSlice(f, numListElems) if raised == nil { if step == 1 { copy(l.elems[start:numListElems-numSliceElems], l.elems[stop:numListElems]) } else { j := 0 for i := start; i != stop; i += step { next := i + step if next > numListElems { next = numListElems } dest := l.elems[i-j : next-j-1] src := l.elems[i+1 : next] copy(dest, src) j++ } } l.elems = l.elems[:numListElems-numSliceElems] } l.mutex.Unlock() return raised } // SetItem sets the index'th element of l to value. func (l *List) SetItem(f *Frame, index int, value *Object) *BaseException { l.mutex.Lock() i, raised := seqCheckedIndex(f, len(l.elems), index) if raised == nil { l.elems[i] = value } l.mutex.Unlock() return raised } // SetSlice replaces the slice of l specified by s with the contents of value // (an iterable). func (l *List) SetSlice(f *Frame, s *Slice, value *Object) *BaseException { l.mutex.Lock() numListElems := len(l.elems) start, stop, step, numSliceElems, raised := s.calcSlice(f, numListElems) if raised == nil { raised = seqApply(f, value, func(elems []*Object, _ bool) *BaseException { numElems := len(elems) if step == 1 { tailElems := l.elems[stop:numListElems] l.resize(numListElems - numSliceElems + numElems) copy(l.elems[start+numElems:], tailElems) copy(l.elems[start:start+numElems], elems) } else if numSliceElems == numElems { i := 0 for j := start; j != stop; j += step { l.elems[j] = elems[i] i++ } } else { format := "attempt to assign sequence of size %d to extended slice of size %d" return f.RaiseType(ValueErrorType, fmt.Sprintf(format, numElems, numSliceElems)) } return nil }) } l.mutex.Unlock() return raised } // Sort reorders l so that its elements are in sorted order. func (l *List) Sort(f *Frame) (raised *BaseException) { l.mutex.RLock() sorter := &listSorter{f, l, nil} defer func() { l.mutex.RUnlock() if val := recover(); val == nil { return } else if s, ok := val.(*listSorter); !ok || s != sorter { panic(val) } raised = sorter.raised }() // Python guarantees stability. See note (9) in: // https://docs.python.org/2/library/stdtypes.html#mutable-sequence-types sort.Stable(sorter) return nil } // resize ensures that len(l.elems) == newLen, reallocating if necessary. // NOTE: l.mutex must be locked when calling resize. func (l *List) resize(newLen int) { if cap(l.elems) < newLen { // Borrowed from CPython's list_resize() in listobject.c. newCap := (newLen >> 3) + 3 + newLen if newLen >= 9 { newCap += 3 } newElems := make([]*Object, len(l.elems), newCap) copy(newElems, l.elems) l.elems = newElems } l.elems = l.elems[:newLen] } // ListType is the object representing the Python 'list' type. var ListType = newBasisType("list", reflect.TypeOf(List{}), toListUnsafe, ObjectType) func listAdd(f *Frame, v, w *Object) (ret *Object, raised *BaseException) { if !w.isInstance(ListType) { return NotImplemented, nil } listV, listW := toListUnsafe(v), toListUnsafe(w) listV.mutex.RLock() listW.mutex.RLock() elems, raised := seqAdd(f, listV.elems, listW.elems) if raised == nil { ret = NewList(elems...).ToObject() } listW.mutex.RUnlock() listV.mutex.RUnlock() return ret, raised } func listAppend(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "append", args, ListType, ObjectType); raised != nil { return nil, raised } toListUnsafe(args[0]).Append(args[1]) return None, nil } func listCount(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "count", args, ListType, ObjectType); raised != nil { return nil, raised } return seqCount(f, args[0], args[1]) } func listDelItem(f *Frame, o *Object, key *Object) *BaseException { l := toListUnsafe(o) if key.isInstance(SliceType) { return l.DelSlice(f, toSliceUnsafe(key)) } if key.typ.slots.Index == nil { format := "list indices must be integers, not %s" return f.RaiseType(TypeErrorType, fmt.Sprintf(format, key.Type().Name())) } index, raised := IndexInt(f, key) if raised != nil { return raised } return l.DelItem(f, index) } func listRemove(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "remove", args, ListType, ObjectType); raised != nil { return nil, raised } value := args[1] l := toListUnsafe(args[0]) l.mutex.Lock() index, raised := seqFindElem(f, l.elems, value) if raised == nil { if index != -1 { l.elems = append(l.elems[:index], l.elems[index+1:]...) } else { raised = f.RaiseType(ValueErrorType, "list.remove(x): x not in list") } } l.mutex.Unlock() if raised != nil { return nil, raised } return None, nil } func listExtend(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) if argc != 2 { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("extend() takes exactly one argument (%d given)", argc)) } return listIAdd(f, args[0], args[1]) } func listContains(f *Frame, l, v *Object) (*Object, *BaseException) { return seqContains(f, l, v) } func listEq(f *Frame, v, w *Object) (*Object, *BaseException) { return listCompare(f, toListUnsafe(v), w, Eq) } func listGE(f *Frame, v, w *Object) (*Object, *BaseException) { return listCompare(f, toListUnsafe(v), w, GE) } func listGetItem(f *Frame, o, key *Object) (*Object, *BaseException) { l := toListUnsafe(o) if key.typ.slots.Index == nil && !key.isInstance(SliceType) { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("list indices must be integers, not %s", key.typ.Name())) } l.mutex.RLock() item, elems, raised := seqGetItem(f, l.elems, key) l.mutex.RUnlock() if raised != nil { return nil, raised } if item != nil { return item, nil } return NewList(elems...).ToObject(), nil } func listGT(f *Frame, v, w *Object) (*Object, *BaseException) { return listCompare(f, toListUnsafe(v), w, GT) } func listIAdd(f *Frame, v, w *Object) (*Object, *BaseException) { l := toListUnsafe(v) raised := seqForEach(f, w, func(o *Object) *BaseException { l.Append(o) return nil }) if raised != nil { return nil, raised } return v, nil } func listIMul(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(IntType) { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("can't multiply sequence by non-int of type '%s'", w.typ.Name())) } l, n := toListUnsafe(v), toIntUnsafe(w).Value() l.mutex.Lock() elems, raised := seqMul(f, l.elems, n) if raised == nil { l.elems = elems } l.mutex.Unlock() if raised != nil { return nil, raised } return v, nil } func listInsert(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "insert", args, ListType, IntType, ObjectType); raised != nil { return nil, raised } l := toListUnsafe(args[0]) l.mutex.Lock() elems := l.elems numElems := len(elems) i := seqClampIndex(toIntUnsafe(args[1]).Value(), numElems) l.resize(numElems + 1) // TODO: The resize() above may have done a copy so we're doing a lot // of extra work here. Optimize this. copy(l.elems[i+1:], elems[i:]) l.elems[i] = args[2] l.mutex.Unlock() return None, nil } func listIter(f *Frame, o *Object) (*Object, *BaseException) { return newListIterator(toListUnsafe(o)), nil } func listLE(f *Frame, v, w *Object) (*Object, *BaseException) { return listCompare(f, toListUnsafe(v), w, LE) } func listLen(f *Frame, o *Object) (*Object, *BaseException) { l := toListUnsafe(o) l.mutex.RLock() ret := NewInt(len(l.elems)).ToObject() l.mutex.RUnlock() return ret, nil } func listNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { elems, raised := seqNew(f, args) if raised != nil { return nil, raised } l := toListUnsafe(newObject(t)) l.elems = elems return l.ToObject(), nil } func listLT(f *Frame, v, w *Object) (*Object, *BaseException) { return listCompare(f, toListUnsafe(v), w, LT) } func listMul(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } l, n := toListUnsafe(v), toIntUnsafe(w).Value() l.mutex.RLock() elems, raised := seqMul(f, l.elems, n) l.mutex.RUnlock() if raised != nil { return nil, raised } return NewList(elems...).ToObject(), nil } func listNE(f *Frame, v, w *Object) (*Object, *BaseException) { return listCompare(f, toListUnsafe(v), w, NE) } func listIndex(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{ListType, ObjectType, ObjectType, ObjectType} argc := len(args) var raised *BaseException if argc == 2 || argc == 3 { expectedTypes = expectedTypes[:argc] } if raised = checkMethodArgs(f, "index", args, expectedTypes...); raised != nil { return nil, raised } l := toListUnsafe(args[0]) l.mutex.RLock() numElems := len(l.elems) start, stop := 0, numElems if argc > 2 { start, raised = IndexInt(f, args[2]) if raised != nil { l.mutex.RUnlock() return nil, raised } } if argc > 3 { stop, raised = IndexInt(f, args[3]) if raised != nil { l.mutex.RUnlock() return nil, raised } } start, stop = adjustIndex(start, stop, numElems) value := args[1] index := -1 if start < numElems && start < stop { index, raised = seqFindElem(f, l.elems[start:stop], value) } l.mutex.RUnlock() if raised != nil { return nil, raised } if index == -1 { return nil, f.RaiseType(ValueErrorType, fmt.Sprintf("%v is not in list", value)) } return NewInt(index + start).ToObject(), nil } func listPop(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) expectedTypes := []*Type{ListType, ObjectType} if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkMethodArgs(f, "pop", args, expectedTypes...); raised != nil { return nil, raised } i := -1 if argc == 2 { var raised *BaseException i, raised = ToIntValue(f, args[1]) if raised != nil { return nil, raised } } l := toListUnsafe(args[0]) l.mutex.Lock() numElems := len(l.elems) if i < 0 { i += numElems } var item *Object var raised *BaseException if i >= numElems || i < 0 { raised = f.RaiseType(IndexErrorType, "list index out of range") } else { item = l.elems[i] l.elems = append(l.elems[:i], l.elems[i+1:]...) } l.mutex.Unlock() return item, raised } func listRepr(f *Frame, o *Object) (*Object, *BaseException) { l := toListUnsafe(o) if f.reprEnter(l.ToObject()) { return NewStr("[...]").ToObject(), nil } l.mutex.RLock() repr, raised := seqRepr(f, l.elems) l.mutex.RUnlock() f.reprLeave(l.ToObject()) if raised != nil { return nil, raised } return NewStr(fmt.Sprintf("[%s]", repr)).ToObject(), nil } func listReverse(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "reverse", args, ListType); raised != nil { return nil, raised } l := toListUnsafe(args[0]) l.mutex.Lock() halfLen := len(l.elems) / 2 for i := 0; i < halfLen; i++ { j := len(l.elems) - i - 1 l.elems[i], l.elems[j] = l.elems[j], l.elems[i] } l.mutex.Unlock() return None, nil } func listSetItem(f *Frame, o, key, value *Object) *BaseException { l := toListUnsafe(o) if key.typ.slots.Index != nil { i, raised := IndexInt(f, key) if raised != nil { return raised } return l.SetItem(f, i, value) } if key.isInstance(SliceType) { return l.SetSlice(f, toSliceUnsafe(key), value) } return f.RaiseType(TypeErrorType, fmt.Sprintf("list indices must be integers, not %s", key.Type().Name())) } func listSort(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { // TODO: Support (cmp=None, key=None, reverse=False) if raised := checkMethodArgs(f, "sort", args, ListType); raised != nil { return nil, raised } l := toListUnsafe(args[0]) l.Sort(f) return None, nil } func initListType(dict map[string]*Object) { dict["append"] = newBuiltinFunction("append", listAppend).ToObject() dict["count"] = newBuiltinFunction("count", listCount).ToObject() dict["extend"] = newBuiltinFunction("extend", listExtend).ToObject() dict["index"] = newBuiltinFunction("index", listIndex).ToObject() dict["insert"] = newBuiltinFunction("insert", listInsert).ToObject() dict["pop"] = newBuiltinFunction("pop", listPop).ToObject() dict["remove"] = newBuiltinFunction("remove", listRemove).ToObject() dict["reverse"] = newBuiltinFunction("reverse", listReverse).ToObject() dict["sort"] = newBuiltinFunction("sort", listSort).ToObject() ListType.slots.Add = &binaryOpSlot{listAdd} ListType.slots.Contains = &binaryOpSlot{listContains} ListType.slots.DelItem = &delItemSlot{listDelItem} ListType.slots.Eq = &binaryOpSlot{listEq} ListType.slots.GE = &binaryOpSlot{listGE} ListType.slots.GetItem = &binaryOpSlot{listGetItem} ListType.slots.GT = &binaryOpSlot{listGT} ListType.slots.Hash = &unaryOpSlot{hashNotImplemented} ListType.slots.IAdd = &binaryOpSlot{listIAdd} ListType.slots.IMul = &binaryOpSlot{listIMul} ListType.slots.Iter = &unaryOpSlot{listIter} ListType.slots.LE = &binaryOpSlot{listLE} ListType.slots.Len = &unaryOpSlot{listLen} ListType.slots.LT = &binaryOpSlot{listLT} ListType.slots.Mul = &binaryOpSlot{listMul} ListType.slots.NE = &binaryOpSlot{listNE} ListType.slots.New = &newSlot{listNew} ListType.slots.Repr = &unaryOpSlot{listRepr} ListType.slots.RMul = &binaryOpSlot{listMul} ListType.slots.SetItem = &setItemSlot{listSetItem} } type listIterator struct { Object list *List mutex sync.Mutex index int } func newListIterator(l *List) *Object { iter := &listIterator{Object: Object{typ: listIteratorType}, list: l} return &iter.Object } func toListIteratorUnsafe(o *Object) *listIterator { return (*listIterator)(o.toPointer()) } var listIteratorType = newBasisType("listiterator", reflect.TypeOf(listIterator{}), toListIteratorUnsafe, ObjectType) func listIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func listIteratorNext(f *Frame, o *Object) (ret *Object, raised *BaseException) { i := toListIteratorUnsafe(o) // Ensure that no mutations happen to the list. i.list.mutex.RLock() i.mutex.Lock() if i.index < len(i.list.elems) { ret = i.list.elems[i.index] i.index++ } else { // Ensure that we raise StopIteration henceforth even if the // sequence grows subsequently. i.index = MaxInt raised = f.Raise(StopIterationType.ToObject(), nil, nil) } i.mutex.Unlock() i.list.mutex.RUnlock() return ret, raised } func initListIteratorType(map[string]*Object) { listIteratorType.flags &= ^(typeFlagBasetype | typeFlagInstantiable) listIteratorType.slots.Iter = &unaryOpSlot{listIteratorIter} listIteratorType.slots.Next = &unaryOpSlot{listIteratorNext} } func listCompare(f *Frame, v *List, w *Object, cmp binaryOpFunc) (*Object, *BaseException) { if !w.isInstance(ListType) { return NotImplemented, nil } listw := toListUnsafe(w) // Order of locking doesn't matter since we're doing a read lock. v.mutex.RLock() listw.mutex.RLock() ret, raised := seqCompare(f, v.elems, listw.elems, cmp) listw.mutex.RUnlock() v.mutex.RUnlock() return ret, raised } type listSorter struct { f *Frame l *List raised *BaseException } func (s *listSorter) Len() int { return len(s.l.elems) } func (s *listSorter) Less(i, j int) bool { lt, raised := LT(s.f, s.l.elems[i], s.l.elems[j]) if raised != nil { s.raised = raised panic(s) } ret, raised := IsTrue(s.f, lt) if raised != nil { s.raised = raised panic(s) } return ret } func (s *listSorter) Swap(i, j int) { s.l.elems[i], s.l.elems[j] = s.l.elems[j], s.l.elems[i] } ================================================ FILE: runtime/list_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "math/big" "reflect" "testing" ) func TestNewList(t *testing.T) { cases := [][]*Object{ nil, []*Object{newObject(ObjectType)}, []*Object{newObject(ObjectType), newObject(ObjectType)}, } for _, args := range cases { l := NewList(args...) if !reflect.DeepEqual(l.elems, args) { t.Errorf("NewList(%v) = %v, want %v", args, l.elems, args) } } } func TestListBinaryOps(t *testing.T) { cases := []struct { fun func(f *Frame, v, w *Object) (*Object, *BaseException) v, w *Object want *Object wantExc *BaseException }{ {Add, newTestList(3).ToObject(), newTestList("foo").ToObject(), newTestList(3, "foo").ToObject(), nil}, {Add, NewList(None).ToObject(), NewList().ToObject(), NewList(None).ToObject(), nil}, {Add, NewList().ToObject(), newObject(ObjectType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'list' and 'object'")}, {Add, None, NewList().ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'NoneType' and 'list'")}, {Mul, NewList().ToObject(), NewInt(10).ToObject(), NewList().ToObject(), nil}, {Mul, newTestList("baz").ToObject(), NewInt(-2).ToObject(), NewList().ToObject(), nil}, {Mul, NewList(None, None).ToObject(), NewInt(0).ToObject(), NewList().ToObject(), nil}, {Mul, newTestList(1, "bar").ToObject(), NewInt(2).ToObject(), newTestList(1, "bar", 1, "bar").ToObject(), nil}, {Mul, NewInt(1).ToObject(), newTestList(1, "bar").ToObject(), newTestList(1, "bar").ToObject(), nil}, {Mul, newObject(ObjectType), NewList(newObject(ObjectType)).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'object' and 'list'")}, {Mul, NewList(newObject(ObjectType)).ToObject(), NewList().ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'list' and 'list'")}, {Mul, NewList(None, None).ToObject(), NewInt(MaxInt).ToObject(), nil, mustCreateException(OverflowErrorType, "result too large")}, } for _, cas := range cases { testCase := invokeTestCase{args: wrapArgs(cas.v, cas.w), want: cas.want, wantExc: cas.wantExc} if err := runInvokeTestCase(wrapFuncForTest(cas.fun), &testCase); err != "" { t.Error(err) } } } func TestListCompare(t *testing.T) { o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(NewList(), NewList()), want: compareAllResultEq}, {args: wrapArgs(newTestList("foo", o), newTestList("foo", o)), want: compareAllResultEq}, {args: wrapArgs(newTestList(4), newTestList(3, 0)), want: compareAllResultGT}, {args: wrapArgs(newTestList(4), newTestList(4, 3, 0)), want: compareAllResultLT}, {args: wrapArgs(NewList(o), NewList()), want: compareAllResultGT}, {args: wrapArgs(NewList(o), newTestList("foo")), want: compareAllResultLT}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestListCount(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewList(), NewInt(1)), want: NewInt(0).ToObject()}, {args: wrapArgs(NewList(None, None, None), None), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestList()), wantExc: mustCreateException(TypeErrorType, "'count' of 'list' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(ListType, "count", &cas); err != "" { t.Error(err) } } } func TestListDelItem(t *testing.T) { badIndexType := newTestClass("badIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(ValueErrorType, "wut") }).ToObject(), })) delItem := mustNotRaise(GetAttr(NewRootFrame(), ListType.ToObject(), NewStr("__delitem__"), nil)) fun := wrapFuncForTest(func(f *Frame, l *List, key *Object) (*Object, *BaseException) { _, raised := delItem.Call(f, wrapArgs(l, key), nil) if raised != nil { return nil, raised } return l.ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(newTestRange(3), 0), want: newTestList(1, 2).ToObject()}, {args: wrapArgs(newTestRange(3), 2), want: newTestList(0, 1).ToObject()}, {args: wrapArgs(NewList(), 101), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(NewList(), newTestSlice(50, 100)), want: NewList().ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1, 3, None)), want: newTestList(1, 4, 5).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1, None, 2)), want: newTestList(1, 3, 5).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(big.NewInt(1), None, 2)), want: newTestList(1, 3, 5).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1, big.NewInt(5), 2)), want: newTestList(1, 3, 5).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1, None, big.NewInt(2))), want: newTestList(1, 3, 5).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1.0, 3, None)), wantExc: mustCreateException(TypeErrorType, errBadSliceIndex)}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(None, None, 4)), want: newTestList(2, 3, 4).ToObject()}, {args: wrapArgs(newTestRange(10), newTestSlice(1, 8, 3)), want: newTestList(0, 2, 3, 5, 6, 8, 9).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3), newTestSlice(1, None, 0)), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, {args: wrapArgs(newTestList(true), None), wantExc: mustCreateException(TypeErrorType, "list indices must be integers, not NoneType")}, {args: wrapArgs(newTestList(true), newObject(badIndexType)), wantExc: mustCreateException(ValueErrorType, "wut")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestListIndex(t *testing.T) { intIndexType := newTestClass("IntIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(0).ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ // {args: wrapArgs(newTestList(), 1, "foo"), wantExc: mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {args: wrapArgs(newTestList(10, 20, 30), 20), want: NewInt(1).ToObject()}, {args: wrapArgs(newTestList(10, 20, 30), 20, newObject(intIndexType)), want: NewInt(1).ToObject()}, {args: wrapArgs(newTestList(0, "foo", "bar"), "foo"), want: NewInt(1).ToObject()}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, 3), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestList(0, 2.0, 2, 3, 4, 2, 1, "foo"), 3, 3), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, 4), wantExc: mustCreateException(ValueErrorType, "3 is not in list")}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, 0, 4), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, 0, 3), wantExc: mustCreateException(ValueErrorType, "3 is not in list")}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, -2), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, -1), wantExc: mustCreateException(ValueErrorType, "3 is not in list")}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, 0, -1), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, 0, -2), wantExc: mustCreateException(ValueErrorType, "3 is not in list")}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, 0, 999), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), "foo", 0, 999), wantExc: mustCreateException(ValueErrorType, "'foo' is not in list")}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, 999), wantExc: mustCreateException(ValueErrorType, "3 is not in list")}, {args: wrapArgs(newTestList(0, 1, 2, 3, 4), 3, 5, 0), wantExc: mustCreateException(ValueErrorType, "3 is not in list")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(ListType, "index", &cas); err != "" { t.Error(err) } } } func TestListRemove(t *testing.T) { fun := newBuiltinFunction("TestListRemove", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { rem, raised := GetAttr(f, ListType.ToObject(), NewStr("remove"), nil) if raised != nil { return nil, raised } if _, raised := rem.Call(f, args, nil); raised != nil { return nil, raised } return args[0], nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(newTestList(1, 2, 3), 2), want: newTestList(1, 3).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 2, 1), 2), want: newTestList(1, 3, 2, 1).ToObject()}, {args: wrapArgs(NewList()), wantExc: mustCreateException(TypeErrorType, "'remove' of 'list' requires 2 arguments")}, {args: wrapArgs(NewList(), 1), wantExc: mustCreateException(ValueErrorType, "list.remove(x): x not in list")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func BenchmarkListContains(b *testing.B) { b.Run("false-3", func(b *testing.B) { t := newTestList("foo", 42, "bar").ToObject() a := wrapArgs(1)[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) b.Run("false-10", func(b *testing.B) { t := newTestList("foo", 42, "bar", "foo", 42, "bar", "foo", 42, "bar", "baz").ToObject() a := wrapArgs(1)[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) b.Run("true-3.1", func(b *testing.B) { t := newTestList("foo", 42, "bar").ToObject() a := wrapArgs("foo")[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) b.Run("true-3.3", func(b *testing.B) { t := newTestList("foo", 42, "bar").ToObject() a := wrapArgs("bar")[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) b.Run("true-10.10", func(b *testing.B) { t := newTestList("foo", 42, "bar", "foo", 42, "bar", "foo", 42, "bar", "baz").ToObject() a := wrapArgs("baz")[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) } func TestListGetItem(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestRange(20), 0), want: NewInt(0).ToObject()}, {args: wrapArgs(newTestRange(20), 19), want: NewInt(19).ToObject()}, {args: wrapArgs(NewList(), 101), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(NewList(), newTestSlice(50, 100)), want: NewList().ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1, 3, None)), want: newTestList(2, 3).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1, None, 2)), want: newTestList(2, 4).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(big.NewInt(1), None, 2)), want: newTestList(2, 4).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1, big.NewInt(5), 2)), want: newTestList(2, 4).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1, None, big.NewInt(2))), want: newTestList(2, 4).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5), newTestSlice(1.0, 3, None)), wantExc: mustCreateException(TypeErrorType, errBadSliceIndex)}, {args: wrapArgs(newTestList(1, 2, 3), newTestSlice(1, None, 0)), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, {args: wrapArgs(newTestList(true), None), wantExc: mustCreateException(TypeErrorType, "list indices must be integers, not NoneType")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(ListType, "__getitem__", &cas); err != "" { t.Error(err) } } } func TestListInplaceOps(t *testing.T) { cases := []struct { fun func(f *Frame, v, w *Object) (*Object, *BaseException) v, w *Object want *Object wantExc *BaseException }{ {IAdd, newTestList(3).ToObject(), newTestList("foo").ToObject(), newTestList(3, "foo").ToObject(), nil}, {IAdd, NewList(None).ToObject(), NewList().ToObject(), NewList(None).ToObject(), nil}, {IAdd, NewList().ToObject(), newObject(ObjectType), nil, mustCreateException(TypeErrorType, "'object' object is not iterable")}, {IMul, NewList().ToObject(), NewInt(10).ToObject(), NewList().ToObject(), nil}, {IMul, newTestList("baz").ToObject(), NewInt(-2).ToObject(), NewList().ToObject(), nil}, {IMul, NewList().ToObject(), None, nil, mustCreateException(TypeErrorType, "can't multiply sequence by non-int of type 'NoneType'")}, } for _, cas := range cases { switch got, result := checkInvokeResult(wrapFuncForTest(cas.fun), []*Object{cas.v, cas.w}, cas.want, cas.wantExc); result { case checkInvokeResultExceptionMismatch: t.Errorf("%s(%v, %v) raised %v, want %v", getFuncName(cas.fun), cas.v, cas.w, got, cas.wantExc) case checkInvokeResultReturnValueMismatch: t.Errorf("%s(%v, %v) = %v, want %v", getFuncName(cas.fun), cas.v, cas.w, got, cas.want) default: if got != nil && got != cas.v { t.Errorf("%s(%v, %v) did not return identity", getFuncName(cas.fun), cas.v, cas.w) } } } } func TestListIsTrue(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewList()), want: False.ToObject()}, {args: wrapArgs(newTestList("foo", None)), want: True.ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(IsTrue), &cas); err != "" { t.Error(err) } } } func TestListAppend(t *testing.T) { fun := newBuiltinFunction("TestListAppend", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestListAppend", args, ListType, ObjectType); raised != nil { return nil, raised } app, raised := GetAttr(f, ListType.ToObject(), NewStr("append"), nil) if raised != nil { return nil, raised } if _, raised := app.Call(f, args, nil); raised != nil { return nil, raised } return args[0], nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(NewList(), None), want: NewList(None).ToObject()}, {args: wrapArgs(NewList(None), 42), want: newTestList(None, 42).ToObject()}, {args: wrapArgs(newTestList(None, 42), "foo"), want: newTestList(None, 42, "foo").ToObject()}, {args: wrapArgs(newTestRange(100), 100), want: newTestRange(101).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestListExtend(t *testing.T) { extend := mustNotRaise(GetAttr(NewRootFrame(), ListType.ToObject(), NewStr("extend"), nil)) fun := newBuiltinFunction("TestListExtend", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if _, raised := extend.Call(f, args, nil); raised != nil { return nil, raised } return args[0], nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(newTestList(), newTestTuple()), want: newTestList().ToObject()}, {args: wrapArgs(newTestList(), newTestList()), want: newTestList().ToObject()}, {args: wrapArgs(newTestList(3), newTestList("foo")), want: newTestList(3, "foo").ToObject()}, {args: wrapArgs(newTestList(), newTestList("foo")), want: newTestList("foo").ToObject()}, {args: wrapArgs(newTestList(3), newTestList()), want: newTestList(3).ToObject()}, {args: wrapArgs(NewStr(""), newTestList()), wantExc: mustCreateException(TypeErrorType, "unbound method extend() must be called with list instance as first argument (got str instance instead)")}, {args: wrapArgs(None, None), wantExc: mustCreateException(TypeErrorType, "unbound method extend() must be called with list instance as first argument (got NoneType instance instead)")}, {args: wrapArgs(newTestList(3), None), wantExc: mustCreateException(TypeErrorType, "'NoneType' object is not iterable")}, {args: wrapArgs(newTestRange(5), newTestList(3)), want: newTestList(0, 1, 2, 3, 4, 3).ToObject()}, {args: wrapArgs(newTestRange(5), newTestList(3)), want: newTestList(0, 1, 2, 3, 4, 3).ToObject()}, {args: wrapArgs(newTestTuple(1, 2, 3), newTestList(3)), wantExc: mustCreateException(TypeErrorType, "unbound method extend() must be called with list instance as first argument (got tuple instance instead)")}, {args: wrapArgs(newTestList(4), newTestTuple(1, 2, 3)), want: newTestList(4, 1, 2, 3).ToObject()}, {args: wrapArgs(newTestList()), wantExc: mustCreateException(TypeErrorType, "extend() takes exactly one argument (1 given)")}, {args: wrapArgs(newTestList(), newTestTuple(), newTestTuple()), wantExc: mustCreateException(TypeErrorType, "extend() takes exactly one argument (3 given)")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestListLen(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewList()), want: NewInt(0).ToObject()}, {args: wrapArgs(NewList(None, None, None)), want: NewInt(3).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(ListType, "__len__", &cas); err != "" { t.Error(err) } } } func TestListNew(t *testing.T) { cases := []invokeTestCase{ {want: NewList().ToObject()}, {args: wrapArgs(newTestTuple(1, 2, 3)), want: newTestList(1, 2, 3).ToObject()}, {args: wrapArgs(newTestDict(1, "foo", "bar", None)), want: newTestList(1, "bar").ToObject()}, {args: wrapArgs(42), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, } for _, cas := range cases { if err := runInvokeTestCase(ListType.ToObject(), &cas); err != "" { t.Error(err) } } } func TestListReverse(t *testing.T) { reverse := mustNotRaise(GetAttr(NewRootFrame(), ListType.ToObject(), NewStr("reverse"), nil)) fun := wrapFuncForTest(func(f *Frame, o *Object, args ...*Object) (*Object, *BaseException) { _, raised := reverse.Call(f, append(Args{o}, args...), nil) if raised != nil { return nil, raised } return o, nil }) cases := []invokeTestCase{ {args: wrapArgs(NewList()), want: NewList().ToObject()}, {args: wrapArgs(newTestList(1, 2, 3)), want: newTestList(3, 2, 1).ToObject()}, {args: wrapArgs(NewList(), 123), wantExc: mustCreateException(TypeErrorType, "'reverse' of 'list' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestListStrRepr(t *testing.T) { recursiveList := newTestList("foo").ToObject() listAppend(NewRootFrame(), []*Object{recursiveList, recursiveList}, nil) cases := []invokeTestCase{ {args: wrapArgs(NewList()), want: NewStr("[]").ToObject()}, {args: wrapArgs(newTestList("foo")), want: NewStr("['foo']").ToObject()}, {args: wrapArgs(newTestList(TupleType, ExceptionType)), want: NewStr("[, ]").ToObject()}, {args: wrapArgs(recursiveList), want: NewStr("['foo', [...]]").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func TestListInsert(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, l *List, args ...*Object) (*Object, *BaseException) { insert, raised := GetAttr(NewRootFrame(), l.ToObject(), NewStr("insert"), nil) if raised != nil { return nil, raised } if _, raised := insert.Call(f, args, nil); raised != nil { return nil, raised } return l.ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(NewList(), 0, None), want: NewList(None).ToObject()}, {args: wrapArgs(newTestList("bar"), -100, "foo"), want: newTestList("foo", "bar").ToObject()}, {args: wrapArgs(newTestList("foo", "bar"), 101, "baz"), want: newTestList("foo", "bar", "baz").ToObject()}, {args: wrapArgs(newTestList("a", "c"), 1, "b"), want: newTestList("a", "b", "c").ToObject()}, {args: wrapArgs(newTestList(1, 2), 0, 0), want: newTestList(0, 1, 2).ToObject()}, {args: wrapArgs(NewList()), wantExc: mustCreateException(TypeErrorType, "'insert' of 'list' requires 3 arguments")}, {args: wrapArgs(NewList(), "foo", 123), wantExc: mustCreateException(TypeErrorType, "'insert' requires a 'int' object but received a 'str'")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestListIter(t *testing.T) { fun := newBuiltinFunction("TestListIter", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "TestListIter", args, ListType); raised != nil { return nil, raised } var got []*Object iter, raised := Iter(f, args[0]) if raised != nil { return nil, raised } raised = seqApply(f, iter, func(elems []*Object, _ bool) *BaseException { got = make([]*Object, len(elems)) copy(got, elems) return nil }) if raised != nil { return nil, raised } return NewList(got...).ToObject(), nil }).ToObject() o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(NewList()), want: NewList().ToObject()}, {args: wrapArgs(newTestList(1, o, "foo")), want: newTestList(1, o, "foo").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestListIteratorIter(t *testing.T) { iter := newListIterator(NewList()) cas := &invokeTestCase{args: wrapArgs(iter), want: iter} if err := runInvokeMethodTestCase(listIteratorType, "__iter__", cas); err != "" { t.Error(err) } } func TestListPop(t *testing.T) { pop := mustNotRaise(GetAttr(NewRootFrame(), ListType.ToObject(), NewStr("pop"), nil)) fun := wrapFuncForTest(func(f *Frame, l *List, args ...*Object) (*Tuple, *BaseException) { result, raised := pop.Call(f, append(Args{l.ToObject()}, args...), nil) if raised != nil { return nil, raised } return newTestTuple(result, l), nil }) cases := []invokeTestCase{ {args: wrapArgs(newTestList(1)), want: newTestTuple(1, newTestList().ToObject()).ToObject()}, {args: wrapArgs(newTestList(1), 0), want: newTestTuple(1, newTestList().ToObject()).ToObject()}, {args: wrapArgs(newTestList(-1, 0, 1)), want: newTestTuple(1, newTestList(-1, 0).ToObject()).ToObject()}, {args: wrapArgs(newTestList(-1, 0, 1), 0), want: newTestTuple(-1, newTestList(0, 1).ToObject()).ToObject()}, {args: wrapArgs(newTestList(-1, 0, 1), NewLong(big.NewInt(1))), want: newTestTuple(0, newTestList(-1, 1).ToObject()).ToObject()}, {args: wrapArgs(newTestList(-1, 0, 1), None), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, {args: wrapArgs(newTestList(-1, 0, 1), None), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, {args: wrapArgs(newTestList(-1, 0, 1), 3), wantExc: mustCreateException(IndexErrorType, "list index out of range")}, {args: wrapArgs(newTestList()), wantExc: mustCreateException(IndexErrorType, "list index out of range")}, {args: wrapArgs(newTestList(), 0), wantExc: mustCreateException(IndexErrorType, "list index out of range")}, {args: wrapArgs(newTestList(), 1), wantExc: mustCreateException(IndexErrorType, "list index out of range")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestListSetItem(t *testing.T) { fun := newBuiltinFunction("TestListSetItem", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { // Check that there is at least one arg, but otherwise leave // the validation to __setitem__. if raised := checkFunctionVarArgs(f, "TestListSetItem", args, ObjectType); raised != nil { return nil, raised } setitem, raised := GetAttr(f, args[0], NewStr("__setitem__"), nil) if raised != nil { return nil, raised } _, raised = setitem.Call(f, args[1:], nil) if raised != nil { return nil, raised } return args[0], nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(newTestList("foo", "bar"), 1, None), want: newTestList("foo", None).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3), newTestSlice(0), newTestList(0)), want: newTestList(0, 1, 2, 3).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3), newTestSlice(1), newTestList(4)), want: newTestList(4, 2, 3).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3), newTestSlice(2, None), newTestList("foo")), want: newTestList(1, 2, "foo").ToObject()}, {args: wrapArgs(newTestList(1, 2, 3), newTestSlice(100, None), newTestList("foo")), want: newTestList(1, 2, 3, "foo").ToObject()}, {args: wrapArgs(newTestList(1, 2, 4, 5), newTestSlice(1, None, 2), newTestTuple("foo", "bar")), want: newTestList(1, "foo", 4, "bar").ToObject()}, {args: wrapArgs(newTestList(1, 2, 3), newTestSlice(None, None, 2), newTestList("foo")), wantExc: mustCreateException(ValueErrorType, "attempt to assign sequence of size 1 to extended slice of size 2")}, {args: wrapArgs(newTestRange(100), newTestSlice(None, None), NewList()), want: NewList().ToObject()}, {args: wrapArgs(NewList(), newTestSlice(4, 8, 0), NewList()), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, {args: wrapArgs(newTestList("foo", "bar"), -100, None), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(NewList(), 101, None), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(newTestList(true), None, false), wantExc: mustCreateException(TypeErrorType, "list indices must be integers, not NoneType")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestListSort(t *testing.T) { sort := mustNotRaise(GetAttr(NewRootFrame(), ListType.ToObject(), NewStr("sort"), nil)) fun := newBuiltinFunction("TestListSort", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if _, raised := sort.Call(f, args, nil); raised != nil { return nil, raised } return args[0], nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(NewList()), want: NewList().ToObject()}, {args: wrapArgs(newTestList("foo", "bar")), want: newTestList("bar", "foo").ToObject()}, {args: wrapArgs(newTestList(true, false)), want: newTestList(false, true).ToObject()}, {args: wrapArgs(newTestList(1, 2, 0, 3)), want: newTestRange(4).ToObject()}, {args: wrapArgs(newTestRange(100)), want: newTestRange(100).ToObject()}, {args: wrapArgs(1), wantExc: mustCreateException(TypeErrorType, "unbound method sort() must be called with list instance as first argument (got int instance instead)")}, {args: wrapArgs(NewList(), 1), wantExc: mustCreateException(TypeErrorType, "'sort' of 'list' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func newTestRange(n int) *List { elems := make([]*Object, n) for i := 0; i < n; i++ { elems[i] = NewInt(i).ToObject() } return NewList(elems...) } func newTestList(elems ...interface{}) *List { listElems, raised := seqWrapEach(NewRootFrame(), elems...) if raised != nil { panic(raised) } return NewList(listElems...) } ================================================ FILE: runtime/long.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "math" "math/big" "reflect" "strings" "sync" ) // By convention in this file, we always use the variable // name "z" or "m" (in the case of DivMod) to refer to // a *big.Int that we intend to modify. For other big.Int // values, we use "x", "y" or other names. Variables z and m // must be allocated within the same function using a big.Int // constructor. // We must never modify the value field of a Long that has // already been made available to the rest of the program, // as this would violate the immutability of Python longs. // Long represents Python 'long' objects. type Long struct { Object value big.Int hashOnce sync.Once hash int } // NewLong returns a new Long holding the given value. func NewLong(x *big.Int) *Long { result := Long{Object: Object{typ: LongType}} result.value.Set(x) return &result } // NewLongFromBytes returns a new Long holding the given bytes, // interpreted as a big endian unsigned integer. func NewLongFromBytes(b []byte) *Long { result := Long{Object: Object{typ: LongType}} result.value.SetBytes(b) return &result } func toLongUnsafe(o *Object) *Long { return (*Long)(o.toPointer()) } // IntValue returns l's value as a plain int if it will not overflow. // Otherwise raises OverflowErrorType. func (l *Long) IntValue(f *Frame) (int, *BaseException) { if !numInIntRange(&l.value) { return 0, f.RaiseType(OverflowErrorType, "Python int too large to convert to a Go int") } return int(l.value.Int64()), nil } // ToObject upcasts l to an Object. func (l *Long) ToObject() *Object { return &l.Object } // Value returns the underlying integer value held by l. func (l *Long) Value() *big.Int { return new(big.Int).Set(&l.value) } // IsTrue returns false if l is zero, true otherwise. func (l *Long) IsTrue() bool { return l.value.Sign() != 0 } // Neg returns a new Long that is the negative of l. func (l *Long) Neg() *Long { result := Long{Object: Object{typ: LongType}} result.value.Set(&l.value) result.value.Neg(&result.value) return &result } // LongType is the object representing the Python 'long' type. var LongType = newBasisType("long", reflect.TypeOf(Long{}), toLongUnsafe, ObjectType) func longAbs(z, x *big.Int) { z.Abs(x) } func longAdd(z, x, y *big.Int) { z.Add(x, y) } func longAnd(z, x, y *big.Int) { z.And(x, y) } func longDiv(z, x, y *big.Int) { m := big.Int{} longDivMod(x, y, z, &m) } func longDivAndMod(z, m, x, y *big.Int) { longDivMod(x, y, z, m) } func longEq(x, y *big.Int) bool { return x.Cmp(y) == 0 } func longGE(x, y *big.Int) bool { return x.Cmp(y) >= 0 } func longGetNewArgs(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__getnewargs__", args, LongType); raised != nil { return nil, raised } return NewTuple1(args[0]).ToObject(), nil } func longGT(x, y *big.Int) bool { return x.Cmp(y) > 0 } func longFloat(f *Frame, o *Object) (*Object, *BaseException) { flt, _ := new(big.Float).SetInt(&toLongUnsafe(o).value).Float64() if math.IsInf(flt, 0) { return nil, f.RaiseType(OverflowErrorType, "long int too large to convert to float") } return NewFloat(flt).ToObject(), nil } func hashBigInt(x *big.Int) int { // TODO: Make this hash match that of cpython. return hashString(x.Text(36)) } func longHex(f *Frame, o *Object) (*Object, *BaseException) { val := numberToBase("0x", 16, o) + "L" return NewStr(val).ToObject(), nil } func longHash(f *Frame, o *Object) (*Object, *BaseException) { l := toLongUnsafe(o) l.hashOnce.Do(func() { // Be compatible with int hashes. if numInIntRange(&l.value) { l.hash = int(l.value.Int64()) } l.hash = hashBigInt(&l.value) }) return NewInt(l.hash).ToObject(), nil } func longIndex(_ *Frame, o *Object) (*Object, *BaseException) { return o, nil } func longInt(f *Frame, o *Object) (*Object, *BaseException) { if l := &toLongUnsafe(o).value; numInIntRange(l) { return NewInt(int(l.Int64())).ToObject(), nil } return o, nil } func longInvert(z, x *big.Int) { z.Not(x) } func longLE(x, y *big.Int) bool { return x.Cmp(y) <= 0 } func longLShift(z, x *big.Int, n uint) { z.Lsh(x, n) } func longLong(f *Frame, o *Object) (*Object, *BaseException) { if o.typ == LongType { return o, nil } l := Long{Object: Object{typ: LongType}} l.value.Set(&toLongUnsafe(o).value) return l.ToObject(), nil } func longLT(x, y *big.Int) bool { return x.Cmp(y) < 0 } func longMul(z, x, y *big.Int) { z.Mul(x, y) } func longMod(m, x, y *big.Int) { z := &big.Int{} longDivMod(x, y, z, m) } func longNative(f *Frame, o *Object) (reflect.Value, *BaseException) { return reflect.ValueOf(toLongUnsafe(o).Value()), nil } func longNE(x, y *big.Int) bool { return x.Cmp(y) != 0 } func longNeg(z, x *big.Int) { z.Neg(x) } func longNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { if t != LongType { // Allocate a plain long and then copy its value into an // object of the long subtype. i, raised := longNew(f, LongType, args, nil) if raised != nil { return nil, raised } result := toLongUnsafe(newObject(t)) result.value = toLongUnsafe(i).value return result.ToObject(), nil } argc := len(args) if argc == 0 { return NewLong(big.NewInt(0)).ToObject(), nil } o := args[0] baseArg := 10 if argc == 1 { if slot := o.typ.slots.Long; slot != nil { result, raised := slot.Fn(f, o) if raised != nil { return nil, raised } if !result.isInstance(LongType) { format := "__long__ returned non-long (type %s)" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, result.typ.Name())) } return result, nil } if raised := checkMethodArgs(f, "__new__", args, StrType); raised != nil { return nil, raised } } else { if raised := checkMethodArgs(f, "__new__", args, StrType, IntType); raised != nil { return nil, raised } baseArg = toIntUnsafe(args[1]).Value() if baseArg != 0 && (baseArg < 2 || baseArg > 36) { return nil, f.RaiseType(ValueErrorType, "long() base must be >= 2 and <= 36") } } s := strings.TrimSpace(toStrUnsafe(o).Value()) if len(s) > 0 && (s[len(s)-1] == 'L' || s[len(s)-1] == 'l') { s = s[:len(s)-1] } base := baseArg if len(s) > 2 { detectedBase := 0 switch s[:2] { case "0b", "0B": detectedBase = 2 case "0o", "0O": detectedBase = 8 case "0x", "0X": detectedBase = 16 } if detectedBase != 0 && (baseArg == 0 || baseArg == detectedBase) { s = s[2:] base = detectedBase } } if base == 0 { base = 10 } i := big.Int{} if _, ok := i.SetString(s, base); !ok { format := "invalid literal for long() with base %d: %s" return nil, f.RaiseType(ValueErrorType, fmt.Sprintf(format, baseArg, toStrUnsafe(o).Value())) } return NewLong(&i).ToObject(), nil } func longNonZero(x *big.Int) bool { return x.Sign() != 0 } func longOct(f *Frame, o *Object) (*Object, *BaseException) { val := numberToBase("0", 8, o) + "L" if val == "00L" { val = "0L" } return NewStr(val).ToObject(), nil } func longOr(z, x, y *big.Int) { z.Or(x, y) } func longPos(z, x *big.Int) { z.Set(x) } func longRepr(f *Frame, o *Object) (*Object, *BaseException) { return NewStr(toLongUnsafe(o).value.Text(10) + "L").ToObject(), nil } func longRShift(z, x *big.Int, n uint) { z.Rsh(x, n) } func longStr(f *Frame, o *Object) (*Object, *BaseException) { return NewStr(toLongUnsafe(o).value.Text(10)).ToObject(), nil } func longSub(z, x, y *big.Int) { z.Sub(x, y) } func longXor(z, x, y *big.Int) { z.Xor(x, y) } func initLongType(dict map[string]*Object) { dict["__getnewargs__"] = newBuiltinFunction("__getnewargs__", longGetNewArgs).ToObject() LongType.slots.Abs = longUnaryOpSlot(longAbs) LongType.slots.Add = longBinaryOpSlot(longAdd) LongType.slots.And = longBinaryOpSlot(longAnd) LongType.slots.Div = longDivModOpSlot(longDiv) LongType.slots.DivMod = longDivAndModOpSlot(longDivAndMod) LongType.slots.Eq = longBinaryBoolOpSlot(longEq) LongType.slots.Float = &unaryOpSlot{longFloat} LongType.slots.FloorDiv = longDivModOpSlot(longDiv) LongType.slots.GE = longBinaryBoolOpSlot(longGE) LongType.slots.GT = longBinaryBoolOpSlot(longGT) LongType.slots.Hash = &unaryOpSlot{longHash} LongType.slots.Hex = &unaryOpSlot{longHex} LongType.slots.Index = &unaryOpSlot{longIndex} LongType.slots.Int = &unaryOpSlot{longInt} LongType.slots.Invert = longUnaryOpSlot(longInvert) LongType.slots.LE = longBinaryBoolOpSlot(longLE) LongType.slots.LShift = longShiftOpSlot(longLShift) LongType.slots.LT = longBinaryBoolOpSlot(longLT) LongType.slots.Long = &unaryOpSlot{longLong} LongType.slots.Mod = longDivModOpSlot(longMod) LongType.slots.Mul = longBinaryOpSlot(longMul) LongType.slots.Native = &nativeSlot{longNative} LongType.slots.NE = longBinaryBoolOpSlot(longNE) LongType.slots.Neg = longUnaryOpSlot(longNeg) LongType.slots.New = &newSlot{longNew} LongType.slots.NonZero = longUnaryBoolOpSlot(longNonZero) LongType.slots.Oct = &unaryOpSlot{longOct} LongType.slots.Or = longBinaryOpSlot(longOr) LongType.slots.Pos = longUnaryOpSlot(longPos) // This operation can return a float, it must use binaryOpSlot directly. LongType.slots.Pow = &binaryOpSlot{longPow} LongType.slots.RAdd = longRBinaryOpSlot(longAdd) LongType.slots.RAnd = longRBinaryOpSlot(longAnd) LongType.slots.RDiv = longRDivModOpSlot(longDiv) LongType.slots.RDivMod = longRDivAndModOpSlot(longDivAndMod) LongType.slots.Repr = &unaryOpSlot{longRepr} LongType.slots.RFloorDiv = longRDivModOpSlot(longDiv) LongType.slots.RMod = longRDivModOpSlot(longMod) LongType.slots.RMul = longRBinaryOpSlot(longMul) LongType.slots.ROr = longRBinaryOpSlot(longOr) LongType.slots.RLShift = longRShiftOpSlot(longLShift) // This operation can return a float, it must use binaryOpSlot directly. LongType.slots.RPow = &binaryOpSlot{longRPow} LongType.slots.RRShift = longRShiftOpSlot(longRShift) LongType.slots.RShift = longShiftOpSlot(longRShift) LongType.slots.RSub = longRBinaryOpSlot(longSub) LongType.slots.RXor = longRBinaryOpSlot(longXor) LongType.slots.Str = &unaryOpSlot{longStr} LongType.slots.Sub = longBinaryOpSlot(longSub) LongType.slots.Xor = longBinaryOpSlot(longXor) } func longCallUnary(fun func(z, x *big.Int), v *Long) *Object { l := Long{Object: Object{typ: LongType}} fun(&l.value, &v.value) return l.ToObject() } func longCallUnaryBool(fun func(x *big.Int) bool, v *Long) *Object { return GetBool(fun(&v.value)).ToObject() } func longCallBinary(fun func(z, x, y *big.Int), v, w *Long) *Object { l := Long{Object: Object{typ: LongType}} fun(&l.value, &v.value, &w.value) return l.ToObject() } func longCallBinaryTuple(fun func(z, m, x, y *big.Int), v, w *Long) *Object { l := Long{Object: Object{typ: LongType}} ll := Long{Object: Object{typ: LongType}} fun(&l.value, &ll.value, &v.value, &w.value) return NewTuple2(l.ToObject(), ll.ToObject()).ToObject() } func longCallBinaryBool(fun func(x, y *big.Int) bool, v, w *Long) *Object { return GetBool(fun(&v.value, &w.value)).ToObject() } func longCallShift(fun func(z, x *big.Int, n uint), f *Frame, v, w *Long) (*Object, *BaseException) { if !numInIntRange(&w.value) { return nil, f.RaiseType(OverflowErrorType, "long int too large to convert to int") } if w.value.Sign() < 0 { return nil, f.RaiseType(ValueErrorType, "negative shift count") } l := Long{Object: Object{typ: LongType}} fun(&l.value, &v.value, uint(w.value.Int64())) return l.ToObject(), nil } func longCallDivMod(fun func(z, x, y *big.Int), f *Frame, v, w *Long) (*Object, *BaseException) { if w.value.Sign() == 0 { return nil, f.RaiseType(ZeroDivisionErrorType, "integer division or modulo by zero") } return longCallBinary(fun, v, w), nil } func longCallDivAndMod(fun func(z, m, x, y *big.Int), f *Frame, v, w *Long) (*Object, *BaseException) { if w.value.Sign() == 0 { return nil, f.RaiseType(ZeroDivisionErrorType, "integer division or modulo by zero") } return longCallBinaryTuple(fun, v, w), nil } func longUnaryOpSlot(fun func(z, x *big.Int)) *unaryOpSlot { f := func(_ *Frame, v *Object) (*Object, *BaseException) { return longCallUnary(fun, toLongUnsafe(v)), nil } return &unaryOpSlot{f} } func longUnaryBoolOpSlot(fun func(x *big.Int) bool) *unaryOpSlot { f := func(_ *Frame, v *Object) (*Object, *BaseException) { return longCallUnaryBool(fun, toLongUnsafe(v)), nil } return &unaryOpSlot{f} } func longBinaryOpSlot(fun func(z, x, y *big.Int)) *binaryOpSlot { f := func(_ *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallBinary(fun, toLongUnsafe(v), toLongUnsafe(w)), nil } return &binaryOpSlot{f} } func longRBinaryOpSlot(fun func(z, x, y *big.Int)) *binaryOpSlot { f := func(_ *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallBinary(fun, toLongUnsafe(w), toLongUnsafe(v)), nil } return &binaryOpSlot{f} } func longDivModOpSlot(fun func(z, x, y *big.Int)) *binaryOpSlot { f := func(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallDivMod(fun, f, toLongUnsafe(v), toLongUnsafe(w)) } return &binaryOpSlot{f} } func longRDivModOpSlot(fun func(z, x, y *big.Int)) *binaryOpSlot { f := func(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallDivMod(fun, f, toLongUnsafe(w), toLongUnsafe(v)) } return &binaryOpSlot{f} } func longDivAndModOpSlot(fun func(z, m, x, y *big.Int)) *binaryOpSlot { f := func(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallDivAndMod(fun, f, toLongUnsafe(v), toLongUnsafe(w)) } return &binaryOpSlot{f} } func longRDivAndModOpSlot(fun func(z, m, x, y *big.Int)) *binaryOpSlot { f := func(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallDivAndMod(fun, f, toLongUnsafe(w), toLongUnsafe(v)) } return &binaryOpSlot{f} } func longShiftOpSlot(fun func(z, x *big.Int, n uint)) *binaryOpSlot { f := func(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallShift(fun, f, toLongUnsafe(v), toLongUnsafe(w)) } return &binaryOpSlot{f} } func longRShiftOpSlot(fun func(z, x *big.Int, n uint)) *binaryOpSlot { f := func(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallShift(fun, f, toLongUnsafe(w), toLongUnsafe(v)) } return &binaryOpSlot{f} } func longBinaryBoolOpSlot(fun func(x, y *big.Int) bool) *binaryOpSlot { f := func(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallBinaryBool(fun, toLongUnsafe(v), toLongUnsafe(w)), nil } return &binaryOpSlot{f} } func longRBinaryBoolOpSlot(fun func(x, y *big.Int) bool) *binaryOpSlot { f := func(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(IntType) { w = intToLong(toIntUnsafe(w)).ToObject() } else if !w.isInstance(LongType) { return NotImplemented, nil } return longCallBinaryBool(fun, toLongUnsafe(w), toLongUnsafe(v)), nil } return &binaryOpSlot{f} } func longPow(f *Frame, v, w *Object) (*Object, *BaseException) { var wLong *big.Int vLong := toLongUnsafe(v).Value() if w.isInstance(LongType) { wLong = toLongUnsafe(w).Value() } else if w.isInstance(IntType) { wLong = big.NewInt(int64(toIntUnsafe(w).Value())) } else { return NotImplemented, nil } if wLong.Sign() < 0 { // The result will be a float, so we call the floating point function. var vFloat, wFloat *Object var raised *BaseException vFloat, raised = longFloat(f, v) if raised != nil { return nil, raised } // w might be an int or a long if w.isInstance(LongType) { wFloat, raised = longFloat(f, w) if raised != nil { return nil, raised } } else if w.isInstance(IntType) { wFloat = NewFloat(float64(toIntUnsafe(w).Value())).ToObject() } else { // This point should not be reachable return nil, f.RaiseType(SystemErrorType, "internal error in longPow") } return floatPow(f, vFloat, wFloat) } return NewLong(big.NewInt(0).Exp(vLong, wLong, nil)).ToObject(), nil } func longRPow(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(LongType) { return longPow(f, w, v) } if w.isInstance(IntType) { wLong := NewLong(big.NewInt(int64(toIntUnsafe(w).Value()))).ToObject() return longPow(f, wLong, v) } return NotImplemented, nil } func longDivMod(x, y, z, m *big.Int) { z.QuoRem(x, y, m) if m.Sign() == -y.Sign() { // In Python the result of the modulo operator is always the // same sign as the divisor, whereas in Go, the result is // always the same sign as the dividend. Therefore we need to // do an adjustment when the sign of the modulo result differs // from that of the divisor. m.Add(m, y) // Relatedly, in Python the result of division truncates toward // negative infinity whereas it truncates toward zero in Go. // The fact that the signs of the divisor and the modulo result // differ implies that the quotient is also negative so we also // adjust the quotient here. z.Sub(z, big.NewInt(1)) } } ================================================ FILE: runtime/long_test.go ================================================ package grumpy import ( "fmt" "math" "math/big" "reflect" "testing" ) var overflowLong = big.NewInt(0).Add(maxIntBig, big.NewInt(101)) func TestLongBasis(t *testing.T) { got := LongType.slots.Basis.Fn(NewLong(big.NewInt(42)).ToObject()).Type() want := reflect.TypeOf(Long{}) if got != want { t.Fatalf("LongType.slots.Basis.Fn(NewLong(big.NewInt(42).ToObject()).Type() = %v, want %v", got, want) } } func TestNewLongFromBytes(t *testing.T) { cases := []struct { bytes []byte want string }{ {bytes: []byte{0x01, 0x00}, want: "100"}, {bytes: []byte{0x01, 0x02, 0x03}, want: "10203"}, {bytes: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09}, want: "10203040506070809"}, } for _, cas := range cases { got := NewLongFromBytes(cas.bytes).value.Text(16) if got != cas.want { t.Errorf("NewLongFromBytes(%v).value.Text(16) = %v, want %v", cas.bytes, got, cas.want) } } } func TestLongReprStr(t *testing.T) { cases := []string{ "0", "123", "-1", "3000", "42", fmt.Sprint(MaxInt), fmt.Sprint(MinInt), "10000000000000000", } for _, cas := range cases { i, _ := new(big.Int).SetString(cas, 0) o := NewLong(i).ToObject() repr, raised := o.typ.slots.Repr.Fn(nil, o) if raised != nil || toStrUnsafe(repr).Value() != cas+"L" { t.Errorf("(%sL).__repr__() = (%v, %v), want (%v, %v)", cas, toStrUnsafe(repr).Value(), raised, cas, nil) } str, raised := o.typ.slots.Str.Fn(nil, o) if raised != nil || toStrUnsafe(str).Value() != cas { t.Errorf("(%sL).__str__() = (%v, %v), want (%v, %v)", cas, toStrUnsafe(str).Value(), raised, cas, nil) } } } func TestLongNew(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__long__": newBuiltinFunction("__long__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return args[0], nil }).ToObject(), })) strictEqType := newTestClassStrictEq("StrictEq", LongType) newStrictEq := func(i *big.Int) *Object { l := Long{Object: Object{typ: strictEqType}} l.value.Set(i) return l.ToObject() } longSubType := newTestClass("LongSubType", []*Type{LongType}, newStringDict(map[string]*Object{})) cases := []invokeTestCase{ {args: wrapArgs(LongType), want: NewLong(big.NewInt(0)).ToObject()}, {args: wrapArgs(LongType, "123"), want: NewLong(big.NewInt(123)).ToObject()}, {args: wrapArgs(LongType, "123L"), want: NewLong(big.NewInt(123)).ToObject()}, {args: wrapArgs(LongType, "123l"), want: NewLong(big.NewInt(123)).ToObject()}, {args: wrapArgs(LongType, " \t123L"), want: NewLong(big.NewInt(123)).ToObject()}, {args: wrapArgs(LongType, "123L \t"), want: NewLong(big.NewInt(123)).ToObject()}, {args: wrapArgs(LongType, "FF", 16), want: NewLong(big.NewInt(255)).ToObject()}, {args: wrapArgs(LongType, "0xFFL", 16), want: NewLong(big.NewInt(255)).ToObject()}, {args: wrapArgs(LongType, "0xE", 0), want: NewLong(big.NewInt(14)).ToObject()}, {args: wrapArgs(LongType, "0b101L", 0), want: NewLong(big.NewInt(5)).ToObject()}, {args: wrapArgs(LongType, "0o726", 0), want: NewLong(big.NewInt(470)).ToObject()}, {args: wrapArgs(LongType, "102", 0), want: NewLong(big.NewInt(102)).ToObject()}, {args: wrapArgs(LongType, 42), want: NewLong(big.NewInt(42)).ToObject()}, {args: wrapArgs(LongType, -3.14), want: NewLong(big.NewInt(-3)).ToObject()}, {args: wrapArgs(LongType, newObject(longSubType)), want: NewLong(big.NewInt(0)).ToObject()}, {args: wrapArgs(strictEqType, big.NewInt(42)), want: newStrictEq(big.NewInt(42))}, {args: wrapArgs(LongType, "0xff"), wantExc: mustCreateException(ValueErrorType, "invalid literal for long() with base 10: 0xff")}, {args: wrapArgs(LongType, ""), wantExc: mustCreateException(ValueErrorType, "invalid literal for long() with base 10: ")}, {args: wrapArgs(LongType, " "), wantExc: mustCreateException(ValueErrorType, "invalid literal for long() with base 10: ")}, {args: wrapArgs(FloatType), wantExc: mustCreateException(TypeErrorType, "long.__new__(float): float is not a subtype of long")}, {args: wrapArgs(LongType, "asldkfj", 1), wantExc: mustCreateException(ValueErrorType, "long() base must be >= 2 and <= 36")}, {args: wrapArgs(LongType, "asldkfj", 37), wantExc: mustCreateException(ValueErrorType, "long() base must be >= 2 and <= 36")}, {args: wrapArgs(LongType, "@#%*(#", 36), wantExc: mustCreateException(ValueErrorType, "invalid literal for long() with base 36: @#%*(#")}, {args: wrapArgs(LongType, "32059823095809238509238590835"), want: NewLong(func() *big.Int { i, _ := new(big.Int).SetString("32059823095809238509238590835", 0); return i }()).ToObject()}, {args: wrapArgs(LongType, big.NewInt(3)), want: NewLong(big.NewInt(3)).ToObject()}, {args: wrapArgs(LongType, NewInt(3)), want: NewLong(big.NewInt(3)).ToObject()}, {args: wrapArgs(LongType, NewInt(3).ToObject()), want: NewLong(big.NewInt(3)).ToObject()}, {args: wrapArgs(LongType, NewLong(big.NewInt(3))), want: NewLong(big.NewInt(3)).ToObject()}, {args: wrapArgs(LongType, NewLong(big.NewInt(3)).ToObject()), want: NewLong(big.NewInt(3)).ToObject()}, {args: wrapArgs(LongType, newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, "'__new__' requires a 'str' object but received a 'object'")}, {args: wrapArgs(LongType, newObject(fooType)), wantExc: mustCreateException(TypeErrorType, "__long__ returned non-long (type Foo)")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(LongType, "__new__", &cas); err != "" { t.Error(err) } } } func TestLongBinaryOps(t *testing.T) { cases := []struct { fun binaryOpFunc v, w interface{} want *Object wantExc *BaseException }{ {Add, -100, 50, NewLong(big.NewInt(-50)).ToObject(), nil}, {Add, newObject(ObjectType), -100, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'object' and 'long'")}, {Add, MaxInt, 1, NewLong(new(big.Int).Add(maxIntBig, big.NewInt(1))).ToObject(), nil}, {And, -100, 50, NewLong(big.NewInt(16)).ToObject(), nil}, {And, MaxInt, MinInt, NewLong(big.NewInt(0)).ToObject(), nil}, {And, newObject(ObjectType), -100, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for &: 'object' and 'long'")}, {Div, 7, 3, NewLong(big.NewInt(2)).ToObject(), nil}, {Div, MaxInt, MinInt, NewLong(big.NewInt(-1)).ToObject(), nil}, {Div, MinInt, MaxInt, NewLong(big.NewInt(-2)).ToObject(), nil}, {Div, NewList().ToObject(), NewLong(big.NewInt(21)).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for /: 'list' and 'long'")}, {Div, 1, 0, nil, mustCreateException(ZeroDivisionErrorType, "integer division or modulo by zero")}, {Div, MinInt, -1, NewLong(new(big.Int).Neg(minIntBig)).ToObject(), nil}, {DivMod, 7, 3, NewTuple2(NewLong(big.NewInt(2)).ToObject(), NewLong(big.NewInt(1)).ToObject()).ToObject(), nil}, {DivMod, 3, -7, NewTuple2(NewLong(big.NewInt(-1)).ToObject(), NewLong(big.NewInt(-4)).ToObject()).ToObject(), nil}, {DivMod, MaxInt, MinInt, NewTuple2(NewLong(big.NewInt(-1)).ToObject(), NewLong(big.NewInt(-1)).ToObject()).ToObject(), nil}, {DivMod, MinInt, MaxInt, NewTuple2(NewLong(big.NewInt(-2)).ToObject(), NewLong(big.NewInt(MaxInt-1)).ToObject()).ToObject(), nil}, {DivMod, MinInt, 1, NewTuple2(NewLong(big.NewInt(MinInt)).ToObject(), NewLong(big.NewInt(0)).ToObject()).ToObject(), nil}, {DivMod, MinInt, -1, NewTuple2(NewLong(new(big.Int).Neg(minIntBig)).ToObject(), NewLong(big.NewInt(0)).ToObject()).ToObject(), nil}, {DivMod, NewList().ToObject(), NewLong(big.NewInt(21)).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for divmod(): 'list' and 'long'")}, {DivMod, 1, 0, nil, mustCreateException(ZeroDivisionErrorType, "integer division or modulo by zero")}, {FloorDiv, 7, 3, NewLong(big.NewInt(2)).ToObject(), nil}, {FloorDiv, MaxInt, MinInt, NewLong(big.NewInt(-1)).ToObject(), nil}, {FloorDiv, MinInt, MaxInt, NewLong(big.NewInt(-2)).ToObject(), nil}, {FloorDiv, NewList().ToObject(), NewLong(big.NewInt(21)).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for //: 'list' and 'long'")}, {FloorDiv, 1, 0, nil, mustCreateException(ZeroDivisionErrorType, "integer division or modulo by zero")}, {FloorDiv, MinInt, -1, NewLong(new(big.Int).Neg(minIntBig)).ToObject(), nil}, {LShift, 2, 4, NewLong(big.NewInt(32)).ToObject(), nil}, {LShift, 12, 10, NewLong(big.NewInt(12288)).ToObject(), nil}, {LShift, 10, 100, NewLong(new(big.Int).Lsh(big.NewInt(10), 100)).ToObject(), nil}, {LShift, 2, -5, nil, mustCreateException(ValueErrorType, "negative shift count")}, {LShift, 4, NewFloat(3.14).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for <<: 'long' and 'float'")}, {LShift, newObject(ObjectType), 4, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for <<: 'object' and 'long'")}, {RShift, 87, 3, NewLong(big.NewInt(10)).ToObject(), nil}, {RShift, -101, 5, NewLong(big.NewInt(-4)).ToObject(), nil}, {RShift, 12, NewInt(10).ToObject(), NewLong(big.NewInt(0)).ToObject(), nil}, {RShift, 12, -10, nil, mustCreateException(ValueErrorType, "negative shift count")}, {RShift, 4, NewFloat(3.14).ToObject(), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for >>: 'long' and 'float'")}, {RShift, newObject(ObjectType), 4, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for >>: 'object' and 'long'")}, {RShift, 4, newObject(ObjectType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for >>: 'long' and 'object'")}, {Mod, 3, -7, NewLong(big.NewInt(-4)).ToObject(), nil}, {Mod, MaxInt, MinInt, NewLong(big.NewInt(-1)).ToObject(), nil}, {Mod, MinInt, MaxInt, NewLong(big.NewInt(int64(MaxInt) - 1)).ToObject(), nil}, {Mod, None, 4, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for %: 'NoneType' and 'long'")}, {Mod, 10, 0, nil, mustCreateException(ZeroDivisionErrorType, "integer division or modulo by zero")}, {Mod, MinInt, 1, NewLong(big.NewInt(0)).ToObject(), nil}, {Mul, 1, 3, NewLong(big.NewInt(3)).ToObject(), nil}, {Mul, newObject(ObjectType), 101, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'object' and 'long'")}, {Mul, int64(4294967295), int64(2147483649), NewLong(new(big.Int).Mul(big.NewInt(4294967295), big.NewInt(2147483649))).ToObject(), nil}, {Or, -100, 50, NewLong(big.NewInt(-66)).ToObject(), nil}, {Or, MaxInt, MinInt, NewLong(big.NewInt(-1)).ToObject(), nil}, {Or, newObject(ObjectType), 100, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for |: 'object' and 'long'")}, {Pow, 2, 128, NewLong(big.NewInt(0).Exp(big.NewInt(2), big.NewInt(128), nil)).ToObject(), nil}, {Pow, 2, -2, NewFloat(0.25).ToObject(), nil}, {Pow, 2, newObject(ObjectType), nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for **: 'long' and 'object'")}, {Sub, 22, 18, NewLong(big.NewInt(4)).ToObject(), nil}, {Sub, IntType.ToObject(), 42, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for -: 'type' and 'long'")}, {Sub, MinInt, 1, NewLong(new(big.Int).Sub(minIntBig, big.NewInt(1))).ToObject(), nil}, {Xor, -100, 50, NewLong(big.NewInt(-82)).ToObject(), nil}, {Xor, MaxInt, MinInt, NewLong(big.NewInt(-1)).ToObject(), nil}, {Xor, newObject(ObjectType), 100, nil, mustCreateException(TypeErrorType, "unsupported operand type(s) for ^: 'object' and 'long'")}, } for _, cas := range cases { v := (*Object)(nil) switch casv := cas.v.(type) { case int: v = NewLong(big.NewInt(int64(casv))).ToObject() case int64: v = NewLong(big.NewInt(casv)).ToObject() case *big.Int: v = NewLong(casv).ToObject() case *Object: v = casv default: t.Errorf("invalid test case: %T", casv) continue } w := (*Object)(nil) switch casw := cas.w.(type) { case int: w = NewLong(big.NewInt(int64(casw))).ToObject() case int64: w = NewLong(big.NewInt(casw)).ToObject() case *big.Int: w = NewLong(casw).ToObject() case *Object: w = casw default: t.Errorf("invalid test case: %T", casw) continue } testCase := invokeTestCase{args: wrapArgs(v, w), want: cas.want, wantExc: cas.wantExc} if err := runInvokeTestCase(wrapFuncForTest(cas.fun), &testCase); err != "" { t.Error(err) } } } func TestLongCompare(t *testing.T) { // Equivalence classes of sample numbers, sorted from least to greatest, nil-separated googol, _ := big.NewFloat(1e100).Int(nil) numbers := []interface{}{ math.Inf(-1), nil, -1e100, new(big.Int).Neg(googol), nil, new(big.Int).Lsh(big.NewInt(-1), 100), nil, // -2^100 MinInt, nil, -306, -306.0, nil, 1, big.NewInt(1), nil, 309683958, big.NewInt(309683958), nil, MaxInt, nil, 1e100, googol, nil, math.Inf(1), nil, } for i, v := range numbers { if v == nil { continue } want := compareAllResultEq for _, w := range numbers[i:] { if w == nil { // switching to a new equivalency class want = compareAllResultLT continue } cas := invokeTestCase{args: wrapArgs(v, w), want: want} if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } } func TestLongInvert(t *testing.T) { googol, _ := big.NewFloat(1e100).Int(nil) cases := []invokeTestCase{ {args: wrapArgs(big.NewInt(2592)), want: NewLong(big.NewInt(-2593)).ToObject()}, {args: wrapArgs(big.NewInt(0)), want: NewLong(big.NewInt(-1)).ToObject()}, {args: wrapArgs(big.NewInt(-43)), want: NewLong(big.NewInt(42)).ToObject()}, {args: wrapArgs(maxIntBig), want: NewLong(minIntBig).ToObject()}, {args: wrapArgs(minIntBig), want: NewLong(maxIntBig).ToObject()}, {args: wrapArgs(googol), want: NewLong(new(big.Int).Not(googol)).ToObject()}, {args: wrapArgs(new(big.Int).Lsh(big.NewInt(-1), 100)), want: NewLong(new(big.Int).Not(new(big.Int).Lsh(big.NewInt(-1), 100))).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(LongType, "__invert__", &cas); err != "" { t.Error(err) } } } func TestLongInt(t *testing.T) { googol, _ := big.NewFloat(1e100).Int(nil) cases := []invokeTestCase{ {args: wrapArgs(big.NewInt(2592)), want: NewInt(2592).ToObject()}, {args: wrapArgs(big.NewInt(0)), want: NewInt(0).ToObject()}, {args: wrapArgs(big.NewInt(-43)), want: NewInt(-43).ToObject()}, {args: wrapArgs(maxIntBig), want: NewInt(MaxInt).ToObject()}, {args: wrapArgs(minIntBig), want: NewInt(MinInt).ToObject()}, {args: wrapArgs(googol), want: NewLong(googol).ToObject()}, {args: wrapArgs(new(big.Int).Lsh(big.NewInt(-1), 100)), want: NewLong(new(big.Int).Lsh(big.NewInt(-1), 100)).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(LongType, "__int__", &cas); err != "" { t.Error(err) } } } func TestLongFloat(t *testing.T) { googol, _ := big.NewFloat(1e100).Int(nil) cases := []invokeTestCase{ {args: wrapArgs(big.NewInt(2592)), want: NewFloat(2592).ToObject()}, {args: wrapArgs(big.NewInt(0)), want: NewFloat(0).ToObject()}, {args: wrapArgs(big.NewInt(-43)), want: NewFloat(-43).ToObject()}, {args: wrapArgs(maxIntBig), want: NewFloat(float64(MaxInt)).ToObject()}, {args: wrapArgs(minIntBig), want: NewFloat(float64(MinInt)).ToObject()}, {args: wrapArgs(googol), want: NewFloat(1e100).ToObject()}, {args: wrapArgs(new(big.Int).Lsh(big.NewInt(-1), 100)), want: NewFloat(-math.Pow(2, 100) + 1).ToObject()}, {args: wrapArgs(new(big.Int).Lsh(big.NewInt(1), 10000)), wantExc: mustCreateException(OverflowErrorType, "long int too large to convert to float")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(LongType, "__float__", &cas); err != "" { t.Error(err) } } } // tests needed: // ✓ arithmetic (long, long) -> long // ✓ add // ✓ sub // ✓ mul // ✓ div // ✓ mod // ✓ boolean logic (long, long) -> long // ✓ and // ✓ or // ✓ xor // ✓ shifts (long, int) -> long // ✓ lsh // ✓ rsh // ✓ comparison (long, long) -> bool // unary ops // hash long -> int // nonzero long -> bool // ✓ invert long -> long // negate long -> long (this slot doesn't exist yet) // ✓ int compatibility // ✓ conversion // ✓ comparison // ✓ float compatibility // ✓ conversion // ✓ comparison // ✓ parsing // ✓ new // ✓ formatting // ✓ repr // ✓ str // native // istrue // native ================================================ FILE: runtime/method.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" ) // Method represents Python 'instancemethod' objects. type Method struct { Object function *Object `attr:"im_func"` self *Object `attr:"im_self"` class *Object `attr:"im_class"` name string `attr:"__name__"` } func toMethodUnsafe(o *Object) *Method { return (*Method)(o.toPointer()) } // ToObject upcasts m to an Object. func (m *Method) ToObject() *Object { return &m.Object } // MethodType is the object representing the Python 'instancemethod' type. var MethodType = newBasisType("instancemethod", reflect.TypeOf(Method{}), toMethodUnsafe, ObjectType) func methodCall(f *Frame, callable *Object, args Args, kwargs KWArgs) (*Object, *BaseException) { m := toMethodUnsafe(callable) argc := len(args) if m.self != nil { methodArgs := f.MakeArgs(argc + 1) methodArgs[0] = m.self copy(methodArgs[1:], args) result, raised := m.function.Call(f, methodArgs, kwargs) f.FreeArgs(methodArgs) return result, raised } if argc < 1 { className, raised := methodGetMemberName(f, m.class) if raised != nil { return nil, raised } format := "unbound method %s() must be called with %s " + "instance as first argument (got nothing instead)" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, m.name, className)) } // instancemethod.__new__ ensures that m.self and m.class are not both // nil. Since m.self is nil, we know that m.class is not. isInst, raised := IsInstance(f, args[0], m.class) if raised != nil { return nil, raised } if !isInst { className, raised := methodGetMemberName(f, m.class) if raised != nil { return nil, raised } format := "unbound method %s() must be called with %s " + "instance as first argument (got %s instance instead)" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, m.name, className, args[0].typ.Name())) } return m.function.Call(f, args, kwargs) } func methodGet(f *Frame, desc, instance *Object, owner *Type) (*Object, *BaseException) { m := toMethodUnsafe(desc) if m.self != nil { // Don't bind a method that's already bound. return desc, nil } if m.class != nil { subcls, raised := IsSubclass(f, owner.ToObject(), m.class) if raised != nil { return nil, raised } if !subcls { // Don't bind if owner is not a subclass of m.class. return desc, nil } } return (&Method{Object{typ: MethodType}, m.function, instance, owner.ToObject(), m.name}).ToObject(), nil } func methodNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{ObjectType, ObjectType, ObjectType} argc := len(args) if argc == 2 { expectedTypes = expectedTypes[:2] } if raised := checkFunctionArgs(f, "__new__", args, expectedTypes...); raised != nil { return nil, raised } function, self := args[0], args[1] if self == None { self = nil } var class *Object if argc > 2 { class = args[2] } else if self == nil { return nil, f.RaiseType(TypeErrorType, "unbound methods must have non-NULL im_class") } if function.Type().slots.Call == nil { return nil, f.RaiseType(TypeErrorType, "first argument must be callable") } functionName, raised := methodGetMemberName(f, function) if raised != nil { return nil, raised } method := &Method{Object{typ: MethodType}, function, self, class, functionName} return method.ToObject(), nil } func methodRepr(f *Frame, o *Object) (*Object, *BaseException) { m := toMethodUnsafe(o) s := "" className, raised := methodGetMemberName(f, m.class) if raised != nil { return nil, raised } functionName, raised := methodGetMemberName(f, m.function) if raised != nil { return nil, raised } if m.self == nil { s = fmt.Sprintf("", className, functionName) } else { repr, raised := Repr(f, m.self) if raised != nil { return nil, raised } s = fmt.Sprintf("", className, functionName, repr.Value()) } return NewStr(s).ToObject(), nil } func initMethodType(map[string]*Object) { MethodType.flags &= ^typeFlagBasetype MethodType.slots.Call = &callSlot{methodCall} MethodType.slots.Get = &getSlot{methodGet} MethodType.slots.Repr = &unaryOpSlot{methodRepr} MethodType.slots.New = &newSlot{methodNew} } func methodGetMemberName(f *Frame, o *Object) (string, *BaseException) { if o == nil { return "?", nil } name, raised := GetAttr(f, o, internedName, None) if raised != nil { return "", raised } if !name.isInstance(BaseStringType) { return "?", nil } nameStr, raised := ToStr(f, name) if raised != nil { return "", raised } return nameStr.Value(), nil } ================================================ FILE: runtime/method_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestMethodCall(t *testing.T) { foo := newBuiltinFunction("foo", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewTuple(args.makeCopy()...).ToObject(), nil }).ToObject() self := newObject(ObjectType) arg0 := NewInt(123).ToObject() arg1 := NewStr("abc").ToObject() cases := []invokeTestCase{ {args: wrapArgs(newTestMethod(foo, self, ObjectType.ToObject())), want: NewTuple(self).ToObject()}, {args: wrapArgs(newTestMethod(foo, None, ObjectType.ToObject()), self), want: NewTuple(self).ToObject()}, {args: wrapArgs(newTestMethod(foo, self, ObjectType.ToObject()), arg0, arg1), want: NewTuple(self, arg0, arg1).ToObject()}, {args: wrapArgs(newTestMethod(foo, None, ObjectType.ToObject()), self, arg0, arg1), want: NewTuple(self, arg0, arg1).ToObject()}, {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "unbound method __call__() must be called with instancemethod instance as first argument (got nothing instead)")}, {args: wrapArgs(newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, "unbound method __call__() must be called with instancemethod instance as first argument (got object instance instead)")}, {args: wrapArgs(newTestMethod(foo, None, IntType.ToObject()), newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, "unbound method foo() must be called with int instance as first argument (got object instance instead)")}, {args: wrapArgs(newTestMethod(foo, None, IntType.ToObject())), wantExc: mustCreateException(TypeErrorType, "unbound method foo() must be called with int instance as first argument (got nothing instead)")}, {args: wrapArgs(newTestMethod(foo, None, None), None), wantExc: mustCreateException(TypeErrorType, "classinfo must be a type or tuple of types")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(MethodType, "__call__", &cas); err != "" { t.Error(err) } } } func TestMethodGet(t *testing.T) { get := mustNotRaise(GetAttr(NewRootFrame(), MethodType.ToObject(), NewStr("__get__"), nil)) fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Object, *BaseException) { o, raised := get.Call(f, args, nil) if raised != nil { return nil, raised } m := toMethodUnsafe(o) self, class := m.self, m.class if self == nil { self = None } if class == nil { class = None } return newTestTuple(m.function, self, class).ToObject(), nil }) dummyFunc := wrapFuncForTest(func() {}) bound := mustNotRaise(MethodType.Call(NewRootFrame(), wrapArgs(dummyFunc, "foo"), nil)) unbound := newTestMethod(dummyFunc, None, IntType.ToObject()) cases := []invokeTestCase{ {args: wrapArgs(bound, "bar", StrType), want: newTestTuple(dummyFunc, "foo", None).ToObject()}, {args: wrapArgs(unbound, "bar", StrType), want: newTestTuple(dummyFunc, None, IntType).ToObject()}, {args: wrapArgs(unbound, 123, IntType), want: newTestTuple(dummyFunc, 123, IntType).ToObject()}, {args: wrapArgs(newTestMethod(dummyFunc, None, None), "bar", StrType), wantExc: mustCreateException(TypeErrorType, "classinfo must be a type or tuple of types")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestMethodNew(t *testing.T) { cases := []invokeTestCase{ {wantExc: mustCreateException(TypeErrorType, "'__new__' requires 3 arguments")}, {args: Args{None, None, None}, wantExc: mustCreateException(TypeErrorType, "first argument must be callable")}, {args: Args{wrapFuncForTest(func() {}), None}, wantExc: mustCreateException(TypeErrorType, "unbound methods must have non-NULL im_class")}, } for _, cas := range cases { if err := runInvokeTestCase(MethodType.ToObject(), &cas); err != "" { t.Error(err) } } } func TestMethodStrRepr(t *testing.T) { foo := newBuiltinFunction("foo", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return None, nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(newTestMethod(foo, None, StrType.ToObject())), want: NewStr("").ToObject()}, {args: wrapArgs(newTestMethod(foo, NewStr("wut").ToObject(), StrType.ToObject())), want: NewStr("").ToObject()}, {args: wrapArgs(newTestMethod(foo, NewInt(123).ToObject(), TupleType.ToObject())), want: NewStr("").ToObject()}, {args: wrapArgs(newTestMethod(foo, None, None)), want: NewStr("").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func newTestMethod(function, self, class *Object) *Method { return toMethodUnsafe(mustNotRaise(MethodType.Call(NewRootFrame(), Args{function, self, class}, nil))) } ================================================ FILE: runtime/module.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "os" "reflect" "runtime/pprof" "strings" "sync" ) type moduleState int const ( moduleStateNew moduleState = iota moduleStateInitializing moduleStateReady ) var ( importMutex sync.Mutex moduleRegistry = map[string]*Code{} // ModuleType is the object representing the Python 'module' type. ModuleType = newBasisType("module", reflect.TypeOf(Module{}), toModuleUnsafe, ObjectType) // SysModules is the global dict of imported modules, aka sys.modules. SysModules = NewDict() ) // Module represents Python 'module' objects. type Module struct { Object mutex recursiveMutex state moduleState code *Code } // ModuleInit functions are called when importing Grumpy modules to execute the // top level code for that module. type ModuleInit func(f *Frame, m *Module) *BaseException // RegisterModule adds the named module to the registry so that it can be // subsequently imported. func RegisterModule(name string, c *Code) { err := "" importMutex.Lock() if moduleRegistry[name] == nil { moduleRegistry[name] = c } else { err = "module already registered: " + name } importMutex.Unlock() if err != "" { logFatal(err) } } // ImportModule takes a fully qualified module name (e.g. a.b.c) and a slice of // code objects where the name of the i'th module is the prefix of name // ending in the i'th dot. The number of dot delimited parts of name must be the // same as the number of code objects. For each successive prefix, ImportModule // looks in sys.modules for an existing module with that name and if not // present creates a new module object, adds it to sys.modules and initializes // it with the corresponding code object. If the module was already present in // sys.modules, it is not re-initialized. The returned slice contains each // package and module initialized in this way in order. // // For example, ImportModule(f, "a.b", []*Code{a.Code, b.Code}) // causes the initialization and entry into sys.modules of Grumpy module a and // then Grumpy module b. The two initialized modules are returned. // // If ImportModule is called in two threads concurrently to import the same // module, both invocations will produce the same module object and the module // is guaranteed to only be initialized once. The second invocation will not // return the module until it is fully initialized. func ImportModule(f *Frame, name string) ([]*Object, *BaseException) { if strings.Contains(name, "/") { o, raised := importOne(f, name) if raised != nil { return nil, raised } return []*Object{o}, nil } parts := strings.Split(name, ".") numParts := len(parts) result := make([]*Object, numParts) var prev *Object for i := 0; i < numParts; i++ { name := strings.Join(parts[:i+1], ".") o, raised := importOne(f, name) if raised != nil { return nil, raised } if prev != nil { if raised := SetAttr(f, prev, NewStr(parts[i]), o); raised != nil { return nil, raised } } result[i] = o prev = o } return result, nil } func importOne(f *Frame, name string) (*Object, *BaseException) { var c *Code // We do very limited locking here resulting in some // sys.modules consistency gotchas. importMutex.Lock() o, raised := SysModules.GetItemString(f, name) if raised == nil && o == nil { if c = moduleRegistry[name]; c == nil { raised = f.RaiseType(ImportErrorType, name) } else { o = newModule(name, c.filename).ToObject() raised = SysModules.SetItemString(f, name, o) } } importMutex.Unlock() if raised != nil { return nil, raised } if o.isInstance(ModuleType) { var raised *BaseException m := toModuleUnsafe(o) m.mutex.Lock(f) if m.state == moduleStateNew { m.state = moduleStateInitializing if _, raised = c.Eval(f, m.Dict(), nil, nil); raised == nil { m.state = moduleStateReady } else { // If the module failed to initialize // then before we relinquish the module // lock, remove it from sys.modules. // Threads waiting on this module will // fail when they don't find it in // sys.modules below. e, tb := f.ExcInfo() if _, raised := SysModules.DelItemString(f, name); raised != nil { f.RestoreExc(e, tb) } } } m.mutex.Unlock(f) if raised != nil { return nil, raised } // The result should be what's in sys.modules, not // necessarily the originally created module since this // is CPython's behavior. o, raised = SysModules.GetItemString(f, name) if raised != nil { return nil, raised } if o == nil { // This can happen in the pathological case // where the module clears itself from // sys.modules during execution and is handled // by CPython in PyImport_ExecCodeModuleEx in // import.c. format := "Loaded module %s not found in sys.modules" return nil, f.RaiseType(ImportErrorType, fmt.Sprintf(format, name)) } } return o, nil } // newModule creates a new Module object with the given fully qualified name // (e.g a.b.c) and its corresponding Python filename. func newModule(name, filename string) *Module { d := newStringDict(map[string]*Object{ "__file__": NewStr(filename).ToObject(), "__name__": NewStr(name).ToObject(), }) return &Module{Object: Object{typ: ModuleType, dict: d}} } func toModuleUnsafe(o *Object) *Module { return (*Module)(o.toPointer()) } // GetFilename returns the __file__ attribute of m, raising SystemError if it // does not exist. func (m *Module) GetFilename(f *Frame) (*Str, *BaseException) { fileAttr, raised := GetAttr(f, m.ToObject(), NewStr("__file__"), None) if raised != nil { return nil, raised } if !fileAttr.isInstance(StrType) { return nil, f.RaiseType(SystemErrorType, "module filename missing") } return toStrUnsafe(fileAttr), nil } // GetName returns the __name__ attribute of m, raising SystemError if it does // not exist. func (m *Module) GetName(f *Frame) (*Str, *BaseException) { nameAttr, raised := GetAttr(f, m.ToObject(), internedName, None) if raised != nil { return nil, raised } if !nameAttr.isInstance(StrType) { return nil, f.RaiseType(SystemErrorType, "nameless module") } return toStrUnsafe(nameAttr), nil } // ToObject upcasts m to an Object. func (m *Module) ToObject() *Object { return &m.Object } func moduleInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{StrType, ObjectType} argc := len(args) if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkFunctionArgs(f, "__init__", args, expectedTypes...); raised != nil { return nil, raised } if raised := SetAttr(f, o, internedName, args[0]); raised != nil { return nil, raised } if argc > 1 { if raised := SetAttr(f, o, NewStr("__doc__"), args[1]); raised != nil { return nil, raised } } return None, nil } func moduleRepr(f *Frame, o *Object) (*Object, *BaseException) { m := toModuleUnsafe(o) name := "?" nameAttr, raised := m.GetName(f) if raised == nil { name = nameAttr.Value() } else { f.RestoreExc(nil, nil) } file := "(built-in)" fileAttr, raised := m.GetFilename(f) if raised == nil { file = fmt.Sprintf("from '%s'", fileAttr.Value()) } else { f.RestoreExc(nil, nil) } return NewStr(fmt.Sprintf("", name, file)).ToObject(), nil } func initModuleType(map[string]*Object) { ModuleType.slots.Init = &initSlot{moduleInit} ModuleType.slots.Repr = &unaryOpSlot{moduleRepr} } // RunMain execs the given code object as a module under the name "__main__". // It handles any exceptions raised during module execution. If no exceptions // were raised then the return value is zero. If a SystemExit was raised then // the return value depends on its code attribute: None -> zero, integer values // are returned as-is. Other code values and exception types produce a return // value of 1. func RunMain(code *Code) int { if file := os.Getenv("GRUMPY_PROFILE"); file != "" { f, err := os.Create(file) if err != nil { logFatal(err.Error()) } if err := pprof.StartCPUProfile(f); err != nil { logFatal(err.Error()) } defer pprof.StopCPUProfile() } m := newModule("__main__", code.filename) m.state = moduleStateInitializing f := NewRootFrame() f.code = code f.globals = m.Dict() if raised := SysModules.SetItemString(f, "__main__", m.ToObject()); raised != nil { Stderr.writeString(raised.String()) } _, e := code.fn(f, nil) if e == nil { return 0 } if !e.isInstance(SystemExitType) { Stderr.writeString(FormatExc(f)) return 1 } f.RestoreExc(nil, nil) o, raised := GetAttr(f, e.ToObject(), NewStr("code"), nil) if raised != nil { return 1 } if o.isInstance(IntType) { return toIntUnsafe(o).Value() } if o == None { return 0 } if s, raised := ToStr(f, o); raised == nil { Stderr.writeString(s.Value() + "\n") } return 1 } ================================================ FILE: runtime/module_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "io/ioutil" "os" "testing" ) func TestImportModule(t *testing.T) { f := NewRootFrame() invalidModule := newObject(ObjectType) foo := newTestModule("foo", "foo/__init__.py") bar := newTestModule("foo.bar", "foo/bar/__init__.py") baz := newTestModule("foo.bar.baz", "foo/bar/baz/__init__.py") qux := newTestModule("foo.qux", "foo/qux/__init__.py") fooCode := NewCode("", "foo/__init__.py", nil, 0, func(*Frame, []*Object) (*Object, *BaseException) { return None, nil }) barCode := NewCode("", "foo/bar/__init__.py", nil, 0, func(*Frame, []*Object) (*Object, *BaseException) { return None, nil }) bazCode := NewCode("", "foo/bar/baz/__init__.py", nil, 0, func(*Frame, []*Object) (*Object, *BaseException) { return None, nil }) quxCode := NewCode("", "foo/qux/__init__.py", nil, 0, func(*Frame, []*Object) (*Object, *BaseException) { return None, nil }) raisesCode := NewCode("", "circular.py", nil, 0, func(f *Frame, _ []*Object) (*Object, *BaseException) { if circularImported { return nil, f.RaiseType(AssertionErrorType, "circular imported recursively") } circularImported = true if _, raised := ImportModule(f, "circular"); raised != nil { return nil, raised } return None, nil }) circularTestModule := newTestModule("circular", "circular.py").ToObject() clearCode := NewCode("", "clear.py", nil, 0, func(f *Frame, _ []*Object) (*Object, *BaseException) { if _, raised := SysModules.DelItemString(f, "clear"); raised != nil { return nil, raised } return None, nil }) // NOTE: This test progressively evolves sys.modules, checking after // each test case that it's populated appropriately. oldSysModules := SysModules oldModuleRegistry := moduleRegistry defer func() { SysModules = oldSysModules moduleRegistry = oldModuleRegistry }() SysModules = newStringDict(map[string]*Object{"invalid": invalidModule}) moduleRegistry = map[string]*Code{ "foo": fooCode, "foo.bar": barCode, "foo.bar.baz": bazCode, "foo.qux": quxCode, "raises": raisesCode, "circular": circularCode, "clear": clearCode, } cases := []struct { name string want *Object wantExc *BaseException wantSysModules *Dict }{ { "noexist", nil, mustCreateException(ImportErrorType, "noexist"), newStringDict(map[string]*Object{"invalid": invalidModule}), }, { "invalid", NewTuple(invalidModule).ToObject(), nil, newStringDict(map[string]*Object{"invalid": invalidModule}), }, { "raises", nil, mustCreateException(ValueErrorType, "uh oh"), newStringDict(map[string]*Object{"invalid": invalidModule}), }, { "foo", NewTuple(foo.ToObject()).ToObject(), nil, newStringDict(map[string]*Object{ "foo": foo.ToObject(), "invalid": invalidModule, }), }, { "foo", NewTuple(foo.ToObject()).ToObject(), nil, newStringDict(map[string]*Object{ "foo": foo.ToObject(), "invalid": invalidModule, }), }, { "foo.qux", NewTuple(foo.ToObject(), qux.ToObject()).ToObject(), nil, newStringDict(map[string]*Object{ "foo": foo.ToObject(), "foo.qux": qux.ToObject(), "invalid": invalidModule, }), }, { "foo.bar.baz", NewTuple(foo.ToObject(), bar.ToObject(), baz.ToObject()).ToObject(), nil, newStringDict(map[string]*Object{ "foo": foo.ToObject(), "foo.bar": bar.ToObject(), "foo.bar.baz": baz.ToObject(), "foo.qux": qux.ToObject(), "invalid": invalidModule, }), }, { "circular", NewTuple(circularTestModule).ToObject(), nil, newStringDict(map[string]*Object{ "circular": circularTestModule, "foo": foo.ToObject(), "foo.bar": bar.ToObject(), "foo.bar.baz": baz.ToObject(), "foo.qux": qux.ToObject(), "invalid": invalidModule, }), }, { "clear", nil, mustCreateException(ImportErrorType, "Loaded module clear not found in sys.modules"), newStringDict(map[string]*Object{ "circular": circularTestModule, "foo": foo.ToObject(), "foo.bar": bar.ToObject(), "foo.bar.baz": baz.ToObject(), "foo.qux": qux.ToObject(), "invalid": invalidModule, }), }, } for _, cas := range cases { mods, raised := ImportModule(f, cas.name) var got *Object if raised == nil { got = NewTuple(mods...).ToObject() } switch checkResult(got, cas.want, raised, cas.wantExc) { case checkInvokeResultExceptionMismatch: t.Errorf("ImportModule(%q) raised %v, want %v", cas.name, raised, cas.wantExc) case checkInvokeResultReturnValueMismatch: t.Errorf("ImportModule(%q) = %v, want %v", cas.name, got, cas.want) } ne := mustNotRaise(NE(f, SysModules.ToObject(), cas.wantSysModules.ToObject())) b, raised := IsTrue(f, ne) if raised != nil { panic(raised) } if b { msg := "ImportModule(%q): sys.modules = %v, want %v" t.Errorf(msg, cas.name, SysModules, cas.wantSysModules) } } } func TestModuleGetNameAndFilename(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, m *Module) (*Tuple, *BaseException) { name, raised := m.GetName(f) if raised != nil { return nil, raised } filename, raised := m.GetFilename(f) if raised != nil { return nil, raised } return newTestTuple(name, filename), nil }) cases := []invokeTestCase{ {args: wrapArgs(newModule("foo", "foo.py")), want: newTestTuple("foo", "foo.py").ToObject()}, {args: Args{mustNotRaise(ModuleType.Call(NewRootFrame(), wrapArgs("foo"), nil))}, wantExc: mustCreateException(SystemErrorType, "module filename missing")}, {args: wrapArgs(&Module{Object: Object{typ: ModuleType, dict: NewDict()}}), wantExc: mustCreateException(SystemErrorType, "nameless module")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestModuleInit(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Tuple, *BaseException) { o, raised := ModuleType.Call(f, args, nil) if raised != nil { return nil, raised } name, raised := GetAttr(f, o, internedName, None) if raised != nil { return nil, raised } doc, raised := GetAttr(f, o, NewStr("__doc__"), None) if raised != nil { return nil, raised } return NewTuple(name, doc), nil }) cases := []invokeTestCase{ {args: wrapArgs("foo"), want: newTestTuple("foo", None).ToObject()}, {args: wrapArgs("foo", 123), want: newTestTuple("foo", 123).ToObject()}, {args: wrapArgs(newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, `'__init__' requires a 'str' object but received a "object"`)}, {wantExc: mustCreateException(TypeErrorType, "'__init__' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestModuleStrRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newModule("foo", "")), want: NewStr("'>").ToObject()}, {args: wrapArgs(newModule("foo.bar.baz", "")), want: NewStr("'>").ToObject()}, {args: Args{mustNotRaise(ModuleType.Call(NewRootFrame(), wrapArgs("foo"), nil))}, want: NewStr("").ToObject()}, {args: wrapArgs(&Module{Object: Object{typ: ModuleType, dict: newTestDict("__file__", "foo.py")}}), want: NewStr("").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func TestRunMain(t *testing.T) { oldSysModules := SysModules defer func() { SysModules = oldSysModules }() cases := []struct { code *Code wantCode int wantOutput string }{ {NewCode("", "test.py", nil, 0, func(*Frame, []*Object) (*Object, *BaseException) { return None, nil }), 0, ""}, {NewCode("", "test.py", nil, 0, func(f *Frame, _ []*Object) (*Object, *BaseException) { return nil, f.Raise(SystemExitType.ToObject(), None, nil) }), 0, ""}, {NewCode("", "test.py", nil, 0, func(f *Frame, _ []*Object) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, "foo") }), 1, "TypeError: foo\n"}, {NewCode("", "test.py", nil, 0, func(f *Frame, _ []*Object) (*Object, *BaseException) { return nil, f.RaiseType(SystemExitType, "foo") }), 1, "foo\n"}, {NewCode("", "test.py", nil, 0, func(f *Frame, _ []*Object) (*Object, *BaseException) { return nil, f.Raise(SystemExitType.ToObject(), NewInt(12).ToObject(), nil) }), 12, ""}, } for _, cas := range cases { SysModules = NewDict() if gotCode, gotOutput, err := runMainAndCaptureStderr(cas.code); err != nil { t.Errorf("runMainRedirectStderr() failed: %v", err) } else if gotCode != cas.wantCode { t.Errorf("RunMain() = %v, want %v", gotCode, cas.wantCode) } else if gotOutput != cas.wantOutput { t.Errorf("RunMain() output %q, want %q", gotOutput, cas.wantOutput) } } } func runMainAndCaptureStderr(code *Code) (int, string, error) { oldStderr := Stderr defer func() { Stderr = oldStderr }() r, w, err := os.Pipe() if err != nil { return 0, "", err } Stderr = NewFileFromFD(w.Fd(), nil) c := make(chan int) go func() { defer w.Close() c <- RunMain(code) }() result := <-c data, err := ioutil.ReadAll(r) if err != nil { return 0, "", err } return result, string(data), nil } var testModuleType *Type func init() { testModuleType, _ = newClass(NewRootFrame(), TypeType, "testModule", []*Type{ModuleType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__eq__", args, ModuleType, ObjectType); raised != nil { return nil, raised } if !args[1].isInstance(ModuleType) { return NotImplemented, nil } m1, m2 := toModuleUnsafe(args[0]), toModuleUnsafe(args[1]) name1, raised := m1.GetName(f) if raised != nil { return nil, raised } name2, raised := m2.GetName(f) if raised != nil { return nil, raised } if name1.Value() != name2.Value() { return False.ToObject(), nil } file1, raised := m1.GetFilename(f) if raised != nil { return nil, raised } file2, raised := m2.GetFilename(f) if raised != nil { return nil, raised } return GetBool(file1.Value() == file2.Value()).ToObject(), nil }).ToObject(), "__ne__": newBuiltinFunction("__ne__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__ne__", args, ModuleType, ObjectType); raised != nil { return nil, raised } eq, raised := Eq(f, args[0], args[1]) if raised != nil { return nil, raised } isEq, raised := IsTrue(f, eq) if raised != nil { return nil, raised } return GetBool(!isEq).ToObject(), nil }).ToObject(), })) } func newTestModule(name, filename string) *Module { return &Module{Object: Object{typ: testModuleType, dict: newTestDict("__name__", name, "__file__", filename)}} } ================================================ FILE: runtime/native.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "bytes" "fmt" "math/big" "reflect" "runtime" "sync" "unsafe" ) var ( nativeBoolMetaclassType = newBasisType("nativebooltype", reflect.TypeOf(nativeBoolMetaclass{}), toNativeBoolMetaclassUnsafe, nativeMetaclassType) nativeFuncType = newSimpleType("func", nativeType) nativeMetaclassType = newBasisType("nativetype", reflect.TypeOf(nativeMetaclass{}), toNativeMetaclassUnsafe, TypeType) nativeSliceType = newSimpleType("slice", nativeType) nativeType = newBasisType("native", reflect.TypeOf(native{}), toNativeUnsafe, ObjectType) // Prepopulate the builtin primitive types so that WrapNative calls on // these kinds of values resolve directly to primitive Python types. nativeTypes = map[reflect.Type]*Type{ reflect.TypeOf(bool(false)): BoolType, reflect.TypeOf(complex64(0)): ComplexType, reflect.TypeOf(complex128(0)): ComplexType, reflect.TypeOf(float32(0)): FloatType, reflect.TypeOf(float64(0)): FloatType, reflect.TypeOf(int(0)): IntType, reflect.TypeOf(int16(0)): IntType, reflect.TypeOf(int32(0)): IntType, reflect.TypeOf(int64(0)): IntType, reflect.TypeOf(int8(0)): IntType, reflect.TypeOf(string("")): StrType, reflect.TypeOf(uint(0)): IntType, reflect.TypeOf(uint16(0)): IntType, reflect.TypeOf(uint32(0)): IntType, reflect.TypeOf(uint64(0)): IntType, reflect.TypeOf(uint8(0)): IntType, reflect.TypeOf(uintptr(0)): IntType, reflect.TypeOf([]rune(nil)): UnicodeType, reflect.TypeOf(big.Int{}): LongType, reflect.TypeOf((*big.Int)(nil)): LongType, } nativeTypesMutex = sync.Mutex{} sliceIteratorType = newBasisType("sliceiterator", reflect.TypeOf(sliceIterator{}), toSliceIteratorUnsafe, ObjectType) ) type nativeMetaclass struct { Type rtype reflect.Type } func toNativeMetaclassUnsafe(o *Object) *nativeMetaclass { return (*nativeMetaclass)(o.toPointer()) } func newNativeType(rtype reflect.Type, base *Type) *Type { t := &nativeMetaclass{ Type{ Object: Object{typ: nativeMetaclassType}, name: nativeTypeName(rtype), basis: base.basis, bases: []*Type{base}, flags: typeFlagDefault, }, rtype, } if !base.isSubclass(nativeType) { t.slots.Native = &nativeSlot{nativeTypedefNative} } return &t.Type } func nativeTypedefNative(f *Frame, o *Object) (reflect.Value, *BaseException) { // The __native__ slot for primitive base classes (e.g. int) returns // the corresponding primitive Go type. For typedef'd primitive types // (e.g. type devNull int) we should return the subtype, not the // primitive type. So first call the primitive type's __native__ and // then convert it to the appropriate subtype. val, raised := o.typ.bases[0].slots.Native.Fn(f, o) if raised != nil { return reflect.Value{}, raised } return val.Convert(toNativeMetaclassUnsafe(o.typ.ToObject()).rtype), nil } func nativeMetaclassNew(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "new", args, nativeMetaclassType); raised != nil { return nil, raised } return WrapNative(f, reflect.New(toNativeMetaclassUnsafe(args[0]).rtype)) } func initNativeMetaclassType(dict map[string]*Object) { nativeMetaclassType.flags &^= typeFlagInstantiable | typeFlagBasetype dict["new"] = newBuiltinFunction("new", nativeMetaclassNew).ToObject() } type nativeBoolMetaclass struct { nativeMetaclass trueValue *Object falseValue *Object } func toNativeBoolMetaclassUnsafe(o *Object) *nativeBoolMetaclass { return (*nativeBoolMetaclass)(o.toPointer()) } func newNativeBoolType(rtype reflect.Type) *Type { t := &nativeBoolMetaclass{ nativeMetaclass: nativeMetaclass{ Type{ Object: Object{typ: nativeBoolMetaclassType}, name: nativeTypeName(rtype), basis: BoolType.basis, bases: []*Type{BoolType}, flags: typeFlagDefault &^ (typeFlagInstantiable | typeFlagBasetype), }, rtype, }, } t.trueValue = (&Int{Object{typ: &t.nativeMetaclass.Type}, 1}).ToObject() t.falseValue = (&Int{Object{typ: &t.nativeMetaclass.Type}, 0}).ToObject() t.slots.Native = &nativeSlot{nativeBoolNative} t.slots.New = &newSlot{nativeBoolNew} return &t.nativeMetaclass.Type } func nativeBoolNative(f *Frame, o *Object) (reflect.Value, *BaseException) { val := reflect.ValueOf(toIntUnsafe(o).IsTrue()) return val.Convert(toNativeMetaclassUnsafe(o.typ.ToObject()).rtype), nil } func nativeBoolNew(f *Frame, t *Type, args Args, kwargs KWArgs) (*Object, *BaseException) { meta := toNativeBoolMetaclassUnsafe(t.ToObject()) argc := len(args) if argc == 0 { return meta.falseValue, nil } if argc != 1 { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("%s() takes at most 1 argument (%d given)", t.Name(), argc)) } ret, raised := IsTrue(f, args[0]) if raised != nil { return nil, raised } if ret { return meta.trueValue, nil } return meta.falseValue, nil } func initNativeBoolMetaclassType(dict map[string]*Object) { nativeBoolMetaclassType.flags &^= typeFlagInstantiable | typeFlagBasetype dict["new"] = newBuiltinFunction("new", nativeMetaclassNew).ToObject() } type native struct { Object value reflect.Value } func toNativeUnsafe(o *Object) *native { return (*native)(o.toPointer()) } // ToObject upcasts n to an Object. func (n *native) ToObject() *Object { return &n.Object } func nativeNative(f *Frame, o *Object) (reflect.Value, *BaseException) { return toNativeUnsafe(o).value, nil } func initNativeType(map[string]*Object) { nativeType.flags = typeFlagDefault &^ typeFlagInstantiable nativeType.slots.Native = &nativeSlot{nativeNative} } func nativeFuncCall(f *Frame, callable *Object, args Args, kwargs KWArgs) (*Object, *BaseException) { return nativeInvoke(f, toNativeUnsafe(callable).value, args) } func nativeFuncGetName(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "_get_name", args, nativeFuncType); raised != nil { return nil, raised } fun := runtime.FuncForPC(toNativeUnsafe(args[0]).value.Pointer()) return NewStr(fun.Name()).ToObject(), nil } func nativeFuncRepr(f *Frame, o *Object) (*Object, *BaseException) { name, raised := GetAttr(f, o, internedName, NewStr("").ToObject()) if raised != nil { return nil, raised } nameStr, raised := ToStr(f, name) if raised != nil { return nil, raised } typeName := nativeTypeName(toNativeUnsafe(o).value.Type()) return NewStr(fmt.Sprintf("<%s %s at %p>", typeName, nameStr.Value(), o)).ToObject(), nil } func initNativeFuncType(dict map[string]*Object) { dict["__name__"] = newProperty(newBuiltinFunction("_get_name", nativeFuncGetName).ToObject(), None, None).ToObject() nativeFuncType.slots.Call = &callSlot{nativeFuncCall} nativeFuncType.slots.Repr = &unaryOpSlot{nativeFuncRepr} } func nativeSliceGetItem(f *Frame, o, key *Object) (*Object, *BaseException) { v := toNativeUnsafe(o).value if key.typ.slots.Index != nil { elem, raised := nativeSliceGetIndex(f, v, key) if raised != nil { return nil, raised } return WrapNative(f, elem) } if !key.isInstance(SliceType) { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("native slice indices must be integers, not %s", key.typ.Name())) } s := toSliceUnsafe(key) start, stop, step, sliceLen, raised := s.calcSlice(f, v.Len()) if raised != nil { return nil, raised } if step == 1 { return WrapNative(f, v.Slice(start, stop)) } result := reflect.MakeSlice(v.Type(), sliceLen, sliceLen) i := 0 for j := start; j != stop; j += step { resultElem := result.Index(i) resultElem.Set(v.Index(j)) i++ } return WrapNative(f, result) } func nativeSliceIter(f *Frame, o *Object) (*Object, *BaseException) { return newSliceIterator(toNativeUnsafe(o).value), nil } func nativeSliceLen(f *Frame, o *Object) (*Object, *BaseException) { return NewInt(toNativeUnsafe(o).value.Len()).ToObject(), nil } func nativeSliceRepr(f *Frame, o *Object) (*Object, *BaseException) { v := toNativeUnsafe(o).value typeName := nativeTypeName(v.Type()) if f.reprEnter(o) { return NewStr(fmt.Sprintf("%s{...}", typeName)).ToObject(), nil } defer f.reprLeave(o) numElems := v.Len() elems := make([]*Object, numElems) for i := 0; i < numElems; i++ { elem, raised := WrapNative(f, v.Index(i)) if raised != nil { return nil, raised } elems[i] = elem } repr, raised := seqRepr(f, elems) if raised != nil { return nil, raised } return NewStr(fmt.Sprintf("%s{%s}", typeName, repr)).ToObject(), nil } func nativeSliceSetItem(f *Frame, o, key, value *Object) *BaseException { v := toNativeUnsafe(o).value elemType := v.Type().Elem() if key.typ.slots.Int != nil { elem, raised := nativeSliceGetIndex(f, v, key) if raised != nil { return raised } if !elem.CanSet() { return f.RaiseType(TypeErrorType, "cannot set slice element") } elemVal, raised := maybeConvertValue(f, value, elemType) if raised != nil { return raised } elem.Set(elemVal) return nil } if key.isInstance(SliceType) { s := toSliceUnsafe(key) start, stop, step, sliceLen, raised := s.calcSlice(f, v.Len()) if raised != nil { return raised } if !v.Index(start).CanSet() { return f.RaiseType(TypeErrorType, "cannot set slice element") } return seqApply(f, value, func(elems []*Object, _ bool) *BaseException { numElems := len(elems) if sliceLen != numElems { format := "attempt to assign sequence of size %d to slice of size %d" return f.RaiseType(ValueErrorType, fmt.Sprintf(format, numElems, sliceLen)) } i := 0 for j := start; j != stop; j += step { elemVal, raised := maybeConvertValue(f, elems[i], elemType) if raised != nil { return raised } v.Index(j).Set(elemVal) i++ } return nil }) } return f.RaiseType(TypeErrorType, fmt.Sprintf("native slice indices must be integers, not %s", key.Type().Name())) } func initNativeSliceType(map[string]*Object) { nativeSliceType.slots.GetItem = &binaryOpSlot{nativeSliceGetItem} nativeSliceType.slots.Iter = &unaryOpSlot{nativeSliceIter} nativeSliceType.slots.Len = &unaryOpSlot{nativeSliceLen} nativeSliceType.slots.Repr = &unaryOpSlot{nativeSliceRepr} nativeSliceType.slots.SetItem = &setItemSlot{nativeSliceSetItem} } func nativeSliceGetIndex(f *Frame, slice reflect.Value, key *Object) (reflect.Value, *BaseException) { i, raised := IndexInt(f, key) if raised != nil { return reflect.Value{}, raised } i, raised = seqCheckedIndex(f, slice.Len(), i) if raised != nil { return reflect.Value{}, raised } return slice.Index(i), nil } type sliceIterator struct { Object slice reflect.Value mutex sync.Mutex numElems int index int } func newSliceIterator(slice reflect.Value) *Object { iter := &sliceIterator{Object: Object{typ: sliceIteratorType}, slice: slice, numElems: slice.Len()} return &iter.Object } func toSliceIteratorUnsafe(o *Object) *sliceIterator { return (*sliceIterator)(o.toPointer()) } func sliceIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func sliceIteratorNext(f *Frame, o *Object) (ret *Object, raised *BaseException) { i := toSliceIteratorUnsafe(o) i.mutex.Lock() if i.index < i.numElems { ret, raised = WrapNative(f, i.slice.Index(i.index)) i.index++ } else { raised = f.Raise(StopIterationType.ToObject(), nil, nil) } i.mutex.Unlock() return ret, raised } func initSliceIteratorType(map[string]*Object) { sliceIteratorType.flags &= ^(typeFlagBasetype | typeFlagInstantiable) sliceIteratorType.slots.Iter = &unaryOpSlot{sliceIteratorIter} sliceIteratorType.slots.Next = &unaryOpSlot{sliceIteratorNext} } // WrapNative takes a reflect.Value object and converts the underlying Go // object to a Python object in the following way: // // - Primitive types are converted in the way you'd expect: Go int types map to // Python int, Go booleans to Python bool, etc. User-defined primitive Go types // are subclasses of the Python primitives. // - *big.Int is represented by Python long. // - Functions are represented by Python type that supports calling into native // functions. // - Interfaces are converted to their concrete held type, or None if IsNil. // - Other native types are wrapped in an opaque native type that does not // support directly accessing the underlying object from Python. When these // opaque objects are passed back into Go by native function calls, however, // they will be unwrapped back to their Go representation. func WrapNative(f *Frame, v reflect.Value) (*Object, *BaseException) { switch v.Kind() { case reflect.Interface: if v.IsNil() { return None, nil } // Interfaces have undefined methods (Method() will return an // invalid func value). What we really want to wrap is the // underlying, concrete object. v = v.Elem() case reflect.Invalid: panic("zero reflect.Value passed to WrapNative") } t := getNativeType(v.Type()) switch v.Kind() { // =============== // Primitive types // =============== // Primitive Go types are translated into primitive Python types or // subclasses of primitive Python types. case reflect.Bool: i := 0 if v.Bool() { i = 1 } // TODO: Make native bool subtypes singletons and add support // for __new__ so we can use t.Call() here. return (&Int{Object{typ: t}, i}).ToObject(), nil case reflect.Complex64: case reflect.Complex128: return t.Call(f, Args{NewComplex(v.Complex()).ToObject()}, nil) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32: return t.Call(f, Args{NewInt(int(v.Int())).ToObject()}, nil) // Handle potentially large ints separately in case of overflow. case reflect.Int64: i := v.Int() if i < int64(MinInt) || i > int64(MaxInt) { return NewLong(big.NewInt(i)).ToObject(), nil } return t.Call(f, Args{NewInt(int(i)).ToObject()}, nil) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: i := v.Uint() if i > uint64(MaxInt) { return t.Call(f, Args{NewLong((new(big.Int).SetUint64(i))).ToObject()}, nil) } return t.Call(f, Args{NewInt(int(i)).ToObject()}, nil) case reflect.Uintptr: // Treat uintptr as a opaque data encoded as a signed integer. i := int64(v.Uint()) if i < int64(MinInt) || i > int64(MaxInt) { return NewLong(big.NewInt(i)).ToObject(), nil } return t.Call(f, Args{NewInt(int(i)).ToObject()}, nil) case reflect.Float32, reflect.Float64: x := v.Float() return t.Call(f, Args{NewFloat(x).ToObject()}, nil) case reflect.String: return t.Call(f, Args{NewStr(v.String()).ToObject()}, nil) case reflect.Slice: if v.Type().Elem() == reflect.TypeOf(rune(0)) { // Avoid reflect.Copy() and Interface()+copy() in case // this is an unexported field. // TODO: Implement a fast path that uses copy() when // v.CanInterface() is true. numRunes := v.Len() runes := make([]rune, numRunes) for i := 0; i < numRunes; i++ { runes[i] = rune(v.Index(i).Int()) } return t.Call(f, Args{NewUnicodeFromRunes(runes).ToObject()}, nil) } // ============= // Complex types // ============= // Non-primitive types are always nativeType subclasses except in a few // specific cases which we handle below. case reflect.Ptr: if v.IsNil() { return None, nil } if v.Type() == reflect.TypeOf((*big.Int)(nil)) { i := v.Interface().(*big.Int) return t.Call(f, Args{NewLong(i).ToObject()}, nil) } if basis := v.Elem(); basisTypes[basis.Type()] != nil { // We have a basis type that is binary compatible with // Object. return (*Object)(unsafe.Pointer(basis.UnsafeAddr())), nil } case reflect.Struct: if i, ok := v.Interface().(big.Int); ok { return t.Call(f, Args{NewLong(&i).ToObject()}, nil) } case reflect.Chan, reflect.Func, reflect.Map: if v.IsNil() { return None, nil } } return (&native{Object{typ: t}, v}).ToObject(), nil } func getNativeType(rtype reflect.Type) *Type { nativeTypesMutex.Lock() t, ok := nativeTypes[rtype] if !ok { // Choose an appropriate base class for this kind of native // object. base := nativeType switch rtype.Kind() { case reflect.Complex64, reflect.Complex128: base = ComplexType case reflect.Float32, reflect.Float64: base = FloatType case reflect.Func: base = nativeFuncType case reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8, reflect.Int, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint8, reflect.Uint, reflect.Uintptr: base = IntType case reflect.Array, reflect.Slice: base = nativeSliceType case reflect.String: base = StrType } d := map[string]*Object{"__module__": builtinStr.ToObject()} numMethod := rtype.NumMethod() for i := 0; i < numMethod; i++ { meth := rtype.Method(i) // A non-empty PkgPath indicates a private method that shouldn't // be registered. if meth.PkgPath == "" { d[meth.Name] = newNativeMethod(meth.Name, meth.Func) } } if rtype.Kind() == reflect.Bool { t = newNativeBoolType(rtype) } else { t = newNativeType(rtype, base) } derefed := rtype for derefed.Kind() == reflect.Ptr { derefed = derefed.Elem() } if derefed.Kind() == reflect.Struct { for i := 0; i < derefed.NumField(); i++ { name := derefed.Field(i).Name d[name] = newNativeField(name, i, t) } } t.setDict(newStringDict(d)) // This cannot fail since we're defining simple classes. if err := prepareType(t); err != "" { logFatal(err) } } nativeTypes[rtype] = t nativeTypesMutex.Unlock() return t } func newNativeField(name string, i int, t *Type) *Object { get := newBuiltinFunction(name, func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, name, args, t); raised != nil { return nil, raised } v := toNativeUnsafe(args[0]).value for v.Type().Kind() == reflect.Ptr { v = v.Elem() } return WrapNative(f, v.Field(i)) }).ToObject() set := newBuiltinFunction(name, func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, name, args, t, ObjectType); raised != nil { return nil, raised } v := toNativeUnsafe(args[0]).value for v.Type().Kind() == reflect.Ptr { v = v.Elem() } field := v.Field(i) if !field.CanSet() { msg := fmt.Sprintf("cannot set field '%s' of type '%s'", name, t.Name()) return nil, f.RaiseType(TypeErrorType, msg) } v, raised := maybeConvertValue(f, args[1], field.Type()) if raised != nil { return nil, raised } field.Set(v) return None, nil }).ToObject() return newProperty(get, set, nil).ToObject() } func newNativeMethod(name string, fun reflect.Value) *Object { return newBuiltinFunction(name, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nativeInvoke(f, fun, args) }).ToObject() } func maybeConvertValue(f *Frame, o *Object, expectedRType reflect.Type) (reflect.Value, *BaseException) { if expectedRType.Kind() == reflect.Ptr { // When the expected type is some basis pointer, check if o is // an instance of that basis and use it if so. if t, ok := basisTypes[expectedRType.Elem()]; ok && o.isInstance(t) { return t.slots.Basis.Fn(o).Addr(), nil } } if o == None { switch expectedRType.Kind() { case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer: return reflect.Zero(expectedRType), nil default: return reflect.Value{}, f.RaiseType(TypeErrorType, fmt.Sprintf("an %s is required", expectedRType)) } } val, raised := ToNative(f, o) if raised != nil { return reflect.Value{}, raised } rtype := val.Type() for { if rtype == expectedRType { return val, nil } if rtype.ConvertibleTo(expectedRType) { return val.Convert(expectedRType), nil } if rtype.Kind() == reflect.Ptr { val = val.Elem() rtype = val.Type() continue } break } return reflect.Value{}, f.RaiseType(TypeErrorType, fmt.Sprintf("an %s is required", expectedRType)) } func nativeFuncTypeName(rtype reflect.Type) string { var buf bytes.Buffer buf.WriteString("func(") numIn := rtype.NumIn() for i := 0; i < numIn; i++ { if i > 0 { buf.WriteString(", ") } buf.WriteString(nativeTypeName(rtype.In(i))) } buf.WriteString(")") numOut := rtype.NumOut() if numOut == 1 { buf.WriteString(" ") buf.WriteString(nativeTypeName(rtype.Out(0))) } else if numOut > 1 { buf.WriteString(" (") for i := 0; i < numOut; i++ { if i > 0 { buf.WriteString(", ") } buf.WriteString(nativeTypeName(rtype.Out(i))) } buf.WriteString(")") } return buf.String() } func nativeInvoke(f *Frame, fun reflect.Value, args Args) (ret *Object, raised *BaseException) { rtype := fun.Type() argc := len(args) expectedArgc := rtype.NumIn() fixedArgc := expectedArgc if rtype.IsVariadic() { fixedArgc-- } if rtype.IsVariadic() && argc < fixedArgc { msg := fmt.Sprintf("native function takes at least %d arguments, (%d given)", fixedArgc, argc) return nil, f.RaiseType(TypeErrorType, msg) } if !rtype.IsVariadic() && argc != fixedArgc { msg := fmt.Sprintf("native function takes %d arguments, (%d given)", fixedArgc, argc) return nil, f.RaiseType(TypeErrorType, msg) } // Convert all the fixed args to their native types. nativeArgs := make([]reflect.Value, argc) for i := 0; i < fixedArgc; i++ { if nativeArgs[i], raised = maybeConvertValue(f, args[i], rtype.In(i)); raised != nil { return nil, raised } } if rtype.IsVariadic() { // The last input in a variadic function is a slice with elem type of the // var args. elementT := rtype.In(fixedArgc).Elem() for i := fixedArgc; i < argc; i++ { if nativeArgs[i], raised = maybeConvertValue(f, args[i], elementT); raised != nil { return nil, raised } } } origExc, origTb := f.RestoreExc(nil, nil) result := fun.Call(nativeArgs) if e, _ := f.ExcInfo(); e != nil { return nil, e } f.RestoreExc(origExc, origTb) numResults := len(result) if numResults > 0 && result[numResults-1].Type() == reflect.TypeOf((*BaseException)(nil)) { numResults-- result = result[:numResults] } // Convert the return value slice to a single value when only one value is // returned, or to a Tuple, when many are returned. switch numResults { case 0: ret = None case 1: ret, raised = WrapNative(f, result[0]) default: elems := make([]*Object, numResults) for i := 0; i < numResults; i++ { if elems[i], raised = WrapNative(f, result[i]); raised != nil { return nil, raised } } ret = NewTuple(elems...).ToObject() } return ret, raised } func nativeTypeName(rtype reflect.Type) string { if rtype.Name() != "" { return rtype.Name() } switch rtype.Kind() { case reflect.Array: return fmt.Sprintf("[%d]%s", rtype.Len(), nativeTypeName(rtype.Elem())) case reflect.Chan: return fmt.Sprintf("chan %s", nativeTypeName(rtype.Elem())) case reflect.Func: return nativeFuncTypeName(rtype) case reflect.Map: return fmt.Sprintf("map[%s]%s", nativeTypeName(rtype.Key()), nativeTypeName(rtype.Elem())) case reflect.Ptr: return fmt.Sprintf("*%s", nativeTypeName(rtype.Elem())) case reflect.Slice: return fmt.Sprintf("[]%s", nativeTypeName(rtype.Elem())) case reflect.Struct: return "anonymous struct" default: return "unknown" } } ================================================ FILE: runtime/native_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "errors" "fmt" "math/big" "reflect" "regexp" "testing" ) func TestNativeMetaclassNew(t *testing.T) { var i int16 intType := newNativeType(reflect.TypeOf(i), IntType) fun := wrapFuncForTest(func(f *Frame, args ...*Object) *BaseException { newFunc, raised := GetAttr(f, intType.ToObject(), NewStr("new"), nil) if raised != nil { return raised } ret, raised := newFunc.Call(f, args, nil) if raised != nil { return raised } got, raised := ToNative(f, ret) if raised != nil { return raised } if got.Type() != reflect.TypeOf(&i) { t.Errorf("%v.new() returned a %s, want a *int16", intType, nativeTypeName(got.Type())) } else if p, ok := got.Interface().(*int16); !ok || p == nil || *p != 0 { t.Errorf("%v.new() returned %v, want &int16(0)", intType, got) } return nil }) cases := []invokeTestCase{ {want: None}, {args: wrapArgs("abc"), wantExc: mustCreateException(TypeErrorType, "'new' of 'nativetype' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestNativeFuncCall(t *testing.T) { cases := []struct { fun interface{} invokeTestCase }{ {func() {}, invokeTestCase{want: None}}, {func() float32 { return 2.0 }, invokeTestCase{want: NewFloat(2.0).ToObject()}}, {func(s string) string { return s }, invokeTestCase{args: wrapArgs("foo"), want: NewStr("foo").ToObject()}}, {func() (int, string) { return 42, "bar" }, invokeTestCase{want: newTestTuple(42, "bar").ToObject()}}, {func(s ...string) int { return len(s) }, invokeTestCase{args: wrapArgs("foo", "bar"), want: NewInt(2).ToObject()}}, {func() {}, invokeTestCase{args: wrapArgs(3.14), wantExc: mustCreateException(TypeErrorType, "native function takes 0 arguments, (1 given)")}}, {func(int, ...string) {}, invokeTestCase{wantExc: mustCreateException(TypeErrorType, "native function takes at least 1 arguments, (0 given)")}}, } for _, cas := range cases { n := &native{Object{typ: nativeFuncType}, reflect.ValueOf(cas.fun)} if err := runInvokeTestCase(n.ToObject(), &cas.invokeTestCase); err != "" { t.Error(err) } } } func TestNativeFuncName(t *testing.T) { re := regexp.MustCompile(`(\w+\.)*\w+$`) fun := wrapFuncForTest(func(f *Frame, o *Object) (string, *BaseException) { desc, raised := GetItem(f, nativeFuncType.Dict().ToObject(), internedName.ToObject()) if raised != nil { return "", raised } get, raised := GetAttr(f, desc, NewStr("__get__"), nil) if raised != nil { return "", raised } name, raised := get.Call(f, wrapArgs(o, nativeFuncType), nil) if raised != nil { return "", raised } if raised := Assert(f, GetBool(name.isInstance(StrType)).ToObject(), nil); raised != nil { return "", raised } return re.FindString(toStrUnsafe(name).Value()), nil }) cases := []invokeTestCase{ {args: wrapArgs(TestNativeFuncName), want: NewStr("grumpy.TestNativeFuncName").ToObject()}, {args: wrapArgs(None), wantExc: mustCreateException(TypeErrorType, "'_get_name' requires a 'func' object but received a 'NoneType'")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestNativeFuncStrRepr(t *testing.T) { cases := []struct { args Args wantPattern string }{ {wrapArgs(TestNativeFuncStrRepr), ``}, {wrapArgs(func() {}), ``}, {wrapArgs(Repr), ``}, } for _, cas := range cases { re := regexp.MustCompile(cas.wantPattern) fun := wrapFuncForTest(func(f *Frame, o *Object) *BaseException { s, raised := ToStr(f, o) if raised != nil { return raised } if !re.MatchString(s.Value()) { t.Errorf("str(%v) = %v, want %v", o, s, re) } s, raised = Repr(f, o) if raised != nil { return raised } if !re.MatchString(s.Value()) { t.Errorf("repr(%v) = %v, want %v", o, s, re) } return nil }) if err := runInvokeTestCase(fun, &invokeTestCase{args: cas.args, want: None}); err != "" { t.Error(err) } } } func TestNativeNew(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, t *Type, args ...*Object) (*Tuple, *BaseException) { o, raised := t.Call(f, args, nil) if raised != nil { return nil, raised } return newTestTuple(o, o.Type()), nil }) type testBool bool testBoolType := getNativeType(reflect.TypeOf(testBool(false))) type testFloat float32 testFloatType := getNativeType(reflect.TypeOf(testFloat(0))) type testString string testStringType := getNativeType(reflect.TypeOf(testString(""))) cases := []invokeTestCase{ {args: wrapArgs(testBoolType), want: newTestTuple(false, testBoolType).ToObject()}, {args: wrapArgs(testBoolType, ""), want: newTestTuple(false, testBoolType).ToObject()}, {args: wrapArgs(testBoolType, 123), want: newTestTuple(true, testBoolType).ToObject()}, {args: wrapArgs(testBoolType, "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "testBool() takes at most 1 argument (2 given)")}, {args: wrapArgs(testFloatType), want: newTestTuple(0.0, testFloatType).ToObject()}, {args: wrapArgs(testFloatType, 3.14), want: newTestTuple(3.14, testFloatType).ToObject()}, {args: wrapArgs(testFloatType, "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "'__new__' of 'float' requires 0 or 1 arguments")}, {args: wrapArgs(testStringType), want: newTestTuple("", testStringType).ToObject()}, {args: wrapArgs(testStringType, "foo"), want: newTestTuple("foo", testStringType).ToObject()}, {args: wrapArgs(testStringType, "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "str() takes at most 1 argument (2 given)")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestNativeSliceIter(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, slice interface{}) (*Object, *BaseException) { o, raised := WrapNative(f, reflect.ValueOf(slice)) if raised != nil { return nil, raised } return TupleType.Call(f, []*Object{o}, nil) }) o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs([]int{}), want: NewTuple().ToObject()}, {args: wrapArgs([]string{"foo", "bar"}), want: newTestTuple("foo", "bar").ToObject()}, {args: wrapArgs([]*Object{True.ToObject(), o}), want: newTestTuple(true, o).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSliceIteratorIter(t *testing.T) { iter := newSliceIterator(reflect.ValueOf([]*Object{})) cas := &invokeTestCase{args: wrapArgs(iter), want: iter} if err := runInvokeMethodTestCase(sliceIteratorType, "__iter__", cas); err != "" { t.Error(err) } } func TestWrapNative(t *testing.T) { o := newObject(ObjectType) d := NewDict() i := 0 n := &native{Object{typ: nativeType}, reflect.ValueOf(&i)} cases := []struct { value interface{} want *Object wantExc *BaseException }{ {true, True.ToObject(), nil}, {True, True.ToObject(), nil}, {123, NewInt(123).ToObject(), nil}, {int8(10), NewInt(10).ToObject(), nil}, {float32(0.5), NewFloat(0.5).ToObject(), nil}, {NewFloat(3.14), NewFloat(3.14).ToObject(), nil}, {uint(MaxInt), NewInt(MaxInt).ToObject(), nil}, {"foobar", NewStr("foobar").ToObject(), nil}, {NewStr("foo"), NewStr("foo").ToObject(), nil}, {uint64(MaxInt) + 100, NewLong(new(big.Int).SetUint64(uint64(MaxInt) + 100)).ToObject(), nil}, {o, o, nil}, {d, d.ToObject(), nil}, {(*Object)(nil), None, nil}, {uintptr(123), NewInt(123).ToObject(), nil}, {n, n.ToObject(), nil}, {(chan int)(nil), None, nil}, {[]rune("hola"), NewUnicode("hola").ToObject(), nil}, {big.NewInt(12345), NewLong(big.NewInt(12345)).ToObject(), nil}, {*big.NewInt(12345), NewLong(big.NewInt(12345)).ToObject(), nil}, } for _, cas := range cases { fun := wrapFuncForTest(func(f *Frame) (*Object, *BaseException) { return WrapNative(f, reflect.ValueOf(cas.value)) }) testCase := invokeTestCase{want: cas.want, wantExc: cas.wantExc} if err := runInvokeTestCase(fun, &testCase); err != "" { t.Error(err) } } } func TestWrapNativeFunc(t *testing.T) { foo := func() int { return 42 } wrappedFoo := mustNotRaise(WrapNative(NewRootFrame(), reflect.ValueOf(foo))) if err := runInvokeTestCase(wrappedFoo, &invokeTestCase{want: NewInt(42).ToObject()}); err != "" { t.Error(err) } } func TestWrapNativeInterface(t *testing.T) { // This seems to be the simplest way to get a reflect.Value that has // Interface kind. iVal := reflect.ValueOf(func() error { return errors.New("foo") }).Call(nil)[0] if iVal.Kind() != reflect.Interface { t.Fatalf("iVal.Kind() = %v, want interface", iVal.Kind()) } o := mustNotRaise(WrapNative(NewRootFrame(), iVal)) cas := &invokeTestCase{args: wrapArgs(o), want: NewStr("foo").ToObject()} if err := runInvokeMethodTestCase(o.typ, "Error", cas); err != "" { t.Error(err) } // Also test the nil interface case. nilVal := reflect.ValueOf(func() error { return nil }).Call(nil)[0] if nilVal.Kind() != reflect.Interface { t.Fatalf("nilVal.Kind() = %v, want interface", nilVal.Kind()) } if o := mustNotRaise(WrapNative(NewRootFrame(), nilVal)); o != None { t.Errorf("WrapNative(%v) = %v, want None", nilVal, o) } } func TestWrapNativeOpaque(t *testing.T) { type fooStruct struct{} foo := &fooStruct{} fooVal := reflect.ValueOf(foo) fun := wrapFuncForTest(func(f *Frame) *BaseException { o, raised := WrapNative(f, fooVal) if raised != nil { return raised } if !o.isInstance(nativeType) { t.Errorf("WrapNative(%v) = %v, want %v", fooVal, o, foo) } else if v := toNativeUnsafe(o).value; v.Type() != reflect.TypeOf(foo) { t.Errorf("WrapNative(%v) = %v, want %v", fooVal, v, foo) } else if got := v.Interface().(*fooStruct); got != foo { t.Errorf("WrapNative(%v) = %v, want %v", fooVal, got, foo) } return nil }) if err := runInvokeTestCase(fun, &invokeTestCase{want: None}); err != "" { t.Error(err) } } func TestGetNativeTypeCaches(t *testing.T) { foo := []struct{}{} typ := getNativeType(reflect.TypeOf(foo)) if got := getNativeType(reflect.TypeOf(foo)); got != typ { t.Errorf("getNativeType(foo) = %v, want %v", got, typ) } } func TestGetNativeTypeFunc(t *testing.T) { if typ := getNativeType(reflect.TypeOf(func() {})); !typ.isSubclass(nativeFuncType) { t.Errorf("getNativeType(func() {}) = %v, want a subclass of func", typ) } else if name := typ.Name(); name != "func()" { t.Errorf(`%v.__name__ == %q, want "func()"`, typ, name) } } type testNativeType struct { data int64 } func (n *testNativeType) Int64() int64 { return n.data } func TestGetNativeTypeMethods(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object) (*Object, *BaseException) { if raised := Assert(f, GetBool(o.isInstance(nativeType)).ToObject(), nil); raised != nil { return nil, raised } int64Method, raised := GetAttr(f, o.Type().ToObject(), NewStr("Int64"), nil) if raised != nil { return nil, raised } return int64Method.Call(f, []*Object{o}, nil) }) cas := invokeTestCase{args: wrapArgs(&testNativeType{12}), want: NewInt(12).ToObject()} if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } func TestGetNativeTypeSlice(t *testing.T) { if typ := getNativeType(reflect.TypeOf([]int{})); !typ.isSubclass(nativeSliceType) { t.Errorf("getNativeType([]int) = %v, want a subclass of slice", typ) } else if name := typ.Name(); name != "[]int" { t.Errorf(`%v.__name__ == %q, want "func()"`, typ, name) } } func TestGetNativeTypeTypedefs(t *testing.T) { type testBool bool type testInt int type testFloat float32 type testString string cases := []struct { rtype reflect.Type super *Type }{ {reflect.TypeOf(testBool(true)), BoolType}, {reflect.TypeOf(testFloat(3.14)), FloatType}, {reflect.TypeOf(testInt(42)), IntType}, {reflect.TypeOf(testString("foo")), StrType}, } for _, cas := range cases { if typ := getNativeType(cas.rtype); typ == cas.super || !typ.isSubclass(cas.super) { t.Errorf("getNativeType(%v) = %v, want a subclass of %v", cas.rtype, typ, cas.super) } } } func TestGetNativeTypeBigInts(t *testing.T) { cases := []struct { rtype reflect.Type typ *Type }{ {reflect.TypeOf(big.Int{}), LongType}, {reflect.TypeOf((*big.Int)(nil)), LongType}, } for _, cas := range cases { if typ := getNativeType(cas.rtype); typ != cas.typ { t.Errorf("getNativeType(%v) = %v, want %v", cas.rtype, typ, cas.typ) } } } func TestMaybeConvertValue(t *testing.T) { type fooStruct struct{} foo := &fooStruct{} fooNative := &native{Object{typ: nativeType}, reflect.ValueOf(&foo)} cases := []struct { o *Object expectedRType reflect.Type want interface{} wantExc *BaseException }{ {NewInt(42).ToObject(), reflect.TypeOf(int(0)), 42, nil}, {NewFloat(0.5).ToObject(), reflect.TypeOf(float32(0)), float32(0.5), nil}, {fooNative.ToObject(), reflect.TypeOf(&fooStruct{}), foo, nil}, {None, reflect.TypeOf((*int)(nil)), (*int)(nil), nil}, {None, reflect.TypeOf(""), nil, mustCreateException(TypeErrorType, "an string is required")}, } for _, cas := range cases { fun := wrapFuncForTest(func(f *Frame) *BaseException { got, raised := maybeConvertValue(f, cas.o, cas.expectedRType) if raised != nil { return raised } if !got.IsValid() || !reflect.DeepEqual(got.Interface(), cas.want) { t.Errorf("maybeConvertValue(%v, %v) = %v, want %v", cas.o, nativeTypeName(cas.expectedRType), got, cas.want) } return nil }) testCase := invokeTestCase{} if cas.wantExc != nil { testCase.wantExc = cas.wantExc } else { testCase.want = None } if err := runInvokeTestCase(fun, &testCase); err != "" { t.Error(err) } } } func TestNativeTypedefNative(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object, wantType reflect.Type) (bool, *BaseException) { val, raised := ToNative(f, o) if raised != nil { return false, raised } return val.Type() == wantType, nil }) type testBool bool testBoolRType := reflect.TypeOf(testBool(false)) type testInt int testIntRType := reflect.TypeOf(testInt(0)) cases := []invokeTestCase{ {args: wrapArgs(mustNotRaise(getNativeType(testBoolRType).Call(NewRootFrame(), wrapArgs(true), nil)), testBoolRType), want: True.ToObject()}, {args: wrapArgs(mustNotRaise(getNativeType(testIntRType).Call(NewRootFrame(), wrapArgs(123), nil)), testIntRType), want: True.ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestNativeTypeName(t *testing.T) { type fooStruct struct{} cases := []struct { rtype reflect.Type want string }{ {reflect.TypeOf([4]int{}), "[4]int"}, {reflect.TypeOf(make(chan *string)), "chan *string"}, {reflect.TypeOf(func() {}), "func()"}, {reflect.TypeOf(func(int, string) {}), "func(int, string)"}, {reflect.TypeOf(func() int { return 0 }), "func() int"}, {reflect.TypeOf(func() (int, float32) { return 0, 0.0 }), "func() (int, float32)"}, {reflect.TypeOf(map[int]fooStruct{}), "map[int]fooStruct"}, {reflect.TypeOf(&fooStruct{}), "*fooStruct"}, {reflect.TypeOf([]byte{}), "[]uint8"}, {reflect.TypeOf(struct{}{}), "anonymous struct"}, } for _, cas := range cases { if got := nativeTypeName(cas.rtype); got != cas.want { t.Errorf("nativeTypeName(%v) = %q, want %q", cas.rtype, got, cas.want) } } } func TestNewNativeFieldChecksInstanceType(t *testing.T) { f := NewRootFrame() // Given a native object native, raised := WrapNative(f, reflect.ValueOf(struct{ foo string }{})) if raised != nil { t.Fatal("Unexpected exception:", raised) } // When its field property is assigned to a different type property, raised := native.typ.Dict().GetItemString(f, "foo") if raised != nil { t.Fatal("Unexpected exception:", raised) } if raised := IntType.Dict().SetItemString(f, "foo", property); raised != nil { t.Fatal("Unexpected exception:", raised) } // And we try to access that property on an object of the new type _, raised = GetAttr(f, NewInt(1).ToObject(), NewStr("foo"), nil) // Then expect a TypeError was raised if raised == nil || raised.Type() != TypeErrorType { t.Fatal("Wanted TypeError; got:", raised) } } func TestNativeSliceGetItem(t *testing.T) { testRange := make([]int, 20) for i := 0; i < len(testRange); i++ { testRange[i] = i } badIndexType := newTestClass("badIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(ValueErrorType, "wut") }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(testRange, 0), want: NewInt(0).ToObject()}, {args: wrapArgs(testRange, 19), want: NewInt(19).ToObject()}, {args: wrapArgs([]struct{}{}, 101), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs([]bool{true}, None), wantExc: mustCreateException(TypeErrorType, "native slice indices must be integers, not NoneType")}, {args: wrapArgs(testRange, newObject(badIndexType)), wantExc: mustCreateException(ValueErrorType, "wut")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(GetItem), &cas); err != "" { t.Error(err) } } } func TestNativeSliceGetItemSlice(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object, slice *Slice, want interface{}) *BaseException { item, raised := GetItem(f, o, slice.ToObject()) if raised != nil { return raised } val, raised := ToNative(f, item) if raised != nil { return raised } v := val.Interface() msg := fmt.Sprintf("%v[%v] = %v, want %v", o, slice, v, want) return Assert(f, GetBool(reflect.DeepEqual(v, want)).ToObject(), NewStr(msg).ToObject()) }) type fooStruct struct { Bar int } cases := []invokeTestCase{ {args: wrapArgs([]string{}, newTestSlice(50, 100), []string{}), want: None}, {args: wrapArgs([]int{1, 2, 3, 4, 5}, newTestSlice(1, 3, None), []int{2, 3}), want: None}, {args: wrapArgs([]fooStruct{fooStruct{1}, fooStruct{10}}, newTestSlice(-1, None, None), []fooStruct{fooStruct{10}}), want: None}, {args: wrapArgs([]int{1, 2, 3, 4, 5}, newTestSlice(1, None, 2), []int{2, 4}), want: None}, {args: wrapArgs([]float64{1.0, 2.0, 3.0, 4.0, 5.0}, newTestSlice(big.NewInt(1), None, 2), []float64{2.0, 4.0}), want: None}, {args: wrapArgs([]string{"1", "2", "3", "4", "5"}, newTestSlice(1, big.NewInt(5), 2), []string{"2", "4"}), want: None}, {args: wrapArgs([]int{1, 2, 3, 4, 5}, newTestSlice(1, None, big.NewInt(2)), []int{2, 4}), want: None}, {args: wrapArgs([]int16{1, 2, 3, 4, 5}, newTestSlice(1.0, 3, None), None), wantExc: mustCreateException(TypeErrorType, errBadSliceIndex)}, {args: wrapArgs([]byte{1, 2, 3}, newTestSlice(1, None, 0), None), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestNativeSliceLen(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs([]string{"foo", "bar"}), want: NewInt(2).ToObject()}, {args: wrapArgs(make([]int, 100)), want: NewInt(100).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(Len), &cas); err != "" { t.Error(err) } } } func TestNativeSliceStrRepr(t *testing.T) { slice := make([]*Object, 2) o := mustNotRaise(WrapNative(NewRootFrame(), reflect.ValueOf(slice))) slice[0] = o slice[1] = NewStr("foo").ToObject() cases := []invokeTestCase{ {args: wrapArgs([]string{"foo", "bar"}), want: NewStr("[]string{'foo', 'bar'}").ToObject()}, {args: wrapArgs([]uint16{123}), want: NewStr("[]uint16{123}").ToObject()}, {args: wrapArgs(o), want: NewStr("[]*Object{[]*Object{...}, 'foo'}").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func TestNativeSliceSetItemSlice(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o, index, value *Object, want interface{}) *BaseException { originalStr := o.String() if raised := SetItem(f, o, index, value); raised != nil { return raised } val, raised := ToNative(f, o) if raised != nil { return raised } v := val.Interface() msg := fmt.Sprintf("%v[%v] = %v -> %v, want %v", originalStr, index, value, o, want) return Assert(f, GetBool(reflect.DeepEqual(v, want)).ToObject(), NewStr(msg).ToObject()) }) type fooStruct struct { bar []int } foo := fooStruct{[]int{1, 2, 3}} bar := mustNotRaise(WrapNative(NewRootFrame(), reflect.ValueOf(foo).Field(0))) cases := []invokeTestCase{ {args: wrapArgs([]string{"foo", "bar"}, 1, "baz", []string{"foo", "baz"}), want: None}, {args: wrapArgs([]uint16{1, 2, 3}, newTestSlice(1), newTestList(4), []uint16{4, 2, 3}), want: None}, {args: wrapArgs([]int{1, 2, 4, 5}, newTestSlice(1, None, 2), newTestTuple(10, 20), []int{1, 10, 4, 20}), want: None}, {args: wrapArgs([]float64{}, newTestSlice(4, 8, 0), NewList(), None), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, {args: wrapArgs([]string{"foo", "bar"}, -100, None, None), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs([]int{}, 101, None, None), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs([]bool{true}, None, false, None), wantExc: mustCreateException(TypeErrorType, "native slice indices must be integers, not NoneType")}, {args: wrapArgs([]int8{1, 2, 3}, newTestSlice(0), []int8{0}, []int8{0, 1, 2, 3}), wantExc: mustCreateException(ValueErrorType, "attempt to assign sequence of size 1 to slice of size 0")}, {args: wrapArgs([]int{1, 2, 3}, newTestSlice(2, None), newTestList("foo"), None), wantExc: mustCreateException(TypeErrorType, "an int is required")}, {args: wrapArgs(bar, 1, 42, None), wantExc: mustCreateException(TypeErrorType, "cannot set slice element")}, {args: wrapArgs(bar, newTestSlice(1), newTestList(42), None), wantExc: mustCreateException(TypeErrorType, "cannot set slice element")}, {args: wrapArgs([]string{"foo", "bar"}, 1, 123.0, None), wantExc: mustCreateException(TypeErrorType, "an string is required")}, {args: wrapArgs([]string{"foo", "bar"}, 1, 123.0, None), wantExc: mustCreateException(TypeErrorType, "an string is required")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestNativeStructFieldGet(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object, attr *Str) (*Object, *BaseException) { return GetAttr(f, o, attr, nil) }) type fooStruct struct { bar int Baz float64 } cases := []invokeTestCase{ {args: wrapArgs(fooStruct{bar: 1}, "bar"), want: NewInt(1).ToObject()}, {args: wrapArgs(&fooStruct{Baz: 3.14}, "Baz"), want: NewFloat(3.14).ToObject()}, {args: wrapArgs(fooStruct{}, "qux"), wantExc: mustCreateException(AttributeErrorType, `'fooStruct' object has no attribute 'qux'`)}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestNativeStructFieldSet(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object, attr *Str, value *Object) (*Object, *BaseException) { if raised := SetAttr(f, o, attr, value); raised != nil { return nil, raised } return GetAttr(f, o, attr, nil) }) type fooStruct struct { bar int Baz float64 } cases := []invokeTestCase{ {args: wrapArgs(&fooStruct{}, "Baz", 1.5), want: NewFloat(1.5).ToObject()}, {args: wrapArgs(fooStruct{}, "bar", 123), wantExc: mustCreateException(TypeErrorType, `cannot set field 'bar' of type 'fooStruct'`)}, {args: wrapArgs(fooStruct{}, "qux", "abc"), wantExc: mustCreateException(AttributeErrorType, `'fooStruct' has no attribute 'qux'`)}, {args: wrapArgs(&fooStruct{}, "Baz", "abc"), wantExc: mustCreateException(TypeErrorType, "an float64 is required")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func wrapArgs(elems ...interface{}) Args { f := NewRootFrame() argc := len(elems) result := make(Args, argc, argc) var raised *BaseException for i, elem := range elems { if result[i], raised = WrapNative(f, reflect.ValueOf(elem)); raised != nil { panic(raised) } } return result } func wrapKWArgs(elems ...interface{}) KWArgs { if len(elems)%2 != 0 { panic("invalid kwargs") } numItems := len(elems) / 2 kwargs := make(KWArgs, numItems, numItems) f := NewRootFrame() for i := 0; i < numItems; i++ { kwargs[i].Name = elems[i*2].(string) kwargs[i].Value = mustNotRaise(WrapNative(f, reflect.ValueOf(elems[i*2+1]))) } return kwargs } ================================================ FILE: runtime/numeric.go ================================================ package grumpy import ( "math/big" "strings" ) const ( // Here we calculate the number of bits in a uint and use that to // create a typeless constant _maxuint which is the largest value that // can be held by a uint. We then use that to create the constants // MaxInt and MinInt below. // Because these constants are typeless, they can be used wherever // a numeric value is needed, without a conversion like int64(). // A typeless number remains untyped when shifted, even if the shift // count is typed. // Start with the two's complement of 0 as a uint which is 0xffff...ff. // This is the number we are after, but it currently has a type (uint). // Dividing it by 0xff gives us 0x0101...01 for the length of a uint. // Taking that mod 15 is effectively counting the ones - one for each // byte in a uint, so we have either 4 or 8. // We multiply by 8 and shift, and now we have 2^32 or 2^64 as a // typeless constant number. // We subtract 1 from that to get maxuint. _maxuint = 1<<(^uint(0)/0xff%15*8) - 1 // MaxInt is the largest (most positive) number that can be stored as an int. MaxInt = _maxuint >> 1 // MinInt is the smallest (most negative) number that can be stored as an int. // The absolute value of MinInt is Maxint+1, thus it can be tricky to deal with. MinInt = -(_maxuint + 1) >> 1 ) var ( maxIntBig = big.NewInt(MaxInt) minIntBig = big.NewInt(MinInt) ) func numParseInteger(z *big.Int, s string, base int) (*big.Int, bool) { s = strings.TrimSpace(s) if len(s) > 2 && s[0] == '0' { switch s[1] { case 'b', 'B': if base == 0 || base == 2 { base = 2 s = s[2:] } case 'o', 'O': if base == 0 || base == 8 { base = 8 s = s[2:] } case 'x', 'X': if base == 0 || base == 16 { base = 16 s = s[2:] } default: base = 8 } } if base == 0 { base = 10 } return z.SetString(s, base) } func numInIntRange(i *big.Int) bool { return i.Cmp(minIntBig) >= 0 && i.Cmp(maxIntBig) <= 0 } ================================================ FILE: runtime/object.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" "sync/atomic" "unsafe" ) var ( objectBasis = reflect.TypeOf(Object{}) objectReconstructorFunc = newBuiltinFunction("_reconstructor", objectReconstructor).ToObject() objectReduceFunc = newBuiltinFunction("__reduce__", objectReduce).ToObject() // ObjectType is the object representing the Python 'object' type. // // We don't use newBasisType() here since that introduces an initialization // cycle between TypeType and ObjectType. ObjectType = &Type{ name: "object", basis: objectBasis, flags: typeFlagDefault, slots: typeSlots{Basis: &basisSlot{objectBasisFunc}}, } ) // Object represents Python 'object' objects. type Object struct { typ *Type `attr:"__class__"` dict *Dict ref *WeakRef } func newObject(t *Type) *Object { var dict *Dict if t != ObjectType { dict = NewDict() } o := (*Object)(unsafe.Pointer(reflect.New(t.basis).Pointer())) o.typ = t o.setDict(dict) return o } // Call invokes the callable Python object o with the given positional and // keyword args. args must be non-nil (but can be empty). kwargs can be nil. func (o *Object) Call(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { call := o.Type().slots.Call if call == nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("'%s' object is not callable", o.Type().Name())) } return call.Fn(f, o, args, kwargs) } // Dict returns o's object dict, aka __dict__. func (o *Object) Dict() *Dict { p := (*unsafe.Pointer)(unsafe.Pointer(&o.dict)) return (*Dict)(atomic.LoadPointer(p)) } func (o *Object) setDict(d *Dict) { p := (*unsafe.Pointer)(unsafe.Pointer(&o.dict)) atomic.StorePointer(p, unsafe.Pointer(d)) } // String returns a string representation of o, e.g. for debugging. func (o *Object) String() string { if o == nil { return "nil" } s, raised := Repr(NewRootFrame(), o) if raised != nil { return fmt.Sprintf("<%s object (repr raised %s)>", o.typ.Name(), raised.typ.Name()) } return s.Value() } // Type returns the Python type of o. func (o *Object) Type() *Type { return o.typ } func (o *Object) toPointer() unsafe.Pointer { return unsafe.Pointer(o) } func (o *Object) isInstance(t *Type) bool { return o.typ.isSubclass(t) } func objectBasisFunc(o *Object) reflect.Value { return reflect.ValueOf(o).Elem() } func objectDelAttr(f *Frame, o *Object, name *Str) *BaseException { desc, raised := o.typ.mroLookup(f, name) if raised != nil { return raised } if desc != nil { if del := desc.Type().slots.Delete; del != nil { return del.Fn(f, desc, o) } } deleted := false d := o.Dict() if d != nil { deleted, raised = d.DelItem(f, name.ToObject()) if raised != nil { return raised } } if !deleted { format := "'%s' object has no attribute '%s'" return f.RaiseType(AttributeErrorType, fmt.Sprintf(format, o.typ.Name(), name.Value())) } return nil } // objectGetAttribute implements the spec here: // https://docs.python.org/2/reference/datamodel.html#invoking-descriptors func objectGetAttribute(f *Frame, o *Object, name *Str) (*Object, *BaseException) { // Look for a data descriptor in the type. var typeGet *getSlot typeAttr, raised := o.typ.mroLookup(f, name) if raised != nil { return nil, raised } if typeAttr != nil { typeGet = typeAttr.typ.slots.Get if typeGet != nil && (typeAttr.typ.slots.Set != nil || typeAttr.typ.slots.Delete != nil) { return typeGet.Fn(f, typeAttr, o, o.Type()) } } // Look in the object's dict. if d := o.Dict(); d != nil { value, raised := d.GetItem(f, name.ToObject()) if value != nil || raised != nil { return value, raised } } // Use the (non-data) descriptor from the type. if typeGet != nil { return typeGet.Fn(f, typeAttr, o, o.Type()) } // Return the ordinary type attribute. if typeAttr != nil { return typeAttr, nil } format := "'%s' object has no attribute '%s'" return nil, f.RaiseType(AttributeErrorType, fmt.Sprintf(format, o.typ.Name(), name.Value())) } func objectHash(f *Frame, o *Object) (*Object, *BaseException) { return NewInt(int(uintptr(o.toPointer()))).ToObject(), nil } func objectNew(f *Frame, t *Type, _ Args, _ KWArgs) (*Object, *BaseException) { if t.flags&typeFlagInstantiable == 0 { format := "object.__new__(%s) is not safe, use %s.__new__()" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, t.Name(), t.Name())) } return newObject(t), nil } func objectReduce(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{ObjectType, IntType} argc := len(args) if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkMethodArgs(f, "__reduce__", args, expectedTypes...); raised != nil { return nil, raised } return objectReduceCommon(f, args) } func objectReduceEx(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{ObjectType, IntType} argc := len(args) if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkMethodArgs(f, "__reduce_ex__", args, expectedTypes...); raised != nil { return nil, raised } reduce, raised := args[0].typ.mroLookup(f, NewStr("__reduce__")) if raised != nil { return nil, raised } if reduce != nil && reduce != objectReduceFunc { // __reduce__ is overridden so prefer using it. return reduce.Call(f, args, nil) } return objectReduceCommon(f, args) } func objectSetAttr(f *Frame, o *Object, name *Str, value *Object) *BaseException { if typeAttr, raised := o.typ.mroLookup(f, name); raised != nil { return raised } else if typeAttr != nil { if typeSet := typeAttr.typ.slots.Set; typeSet != nil { return typeSet.Fn(f, typeAttr, o, value) } } if d := o.Dict(); d != nil { if raised := d.SetItem(f, name.ToObject(), value); raised == nil || !raised.isInstance(KeyErrorType) { return nil } } return f.RaiseType(AttributeErrorType, fmt.Sprintf("'%s' has no attribute '%s'", o.typ.Name(), name.Value())) } func initObjectType(dict map[string]*Object) { ObjectType.typ = TypeType dict["__reduce__"] = objectReduceFunc dict["__reduce_ex__"] = newBuiltinFunction("__reduce_ex__", objectReduceEx).ToObject() dict["__dict__"] = newProperty(newBuiltinFunction("_get_dict", objectGetDict).ToObject(), newBuiltinFunction("_set_dict", objectSetDict).ToObject(), nil).ToObject() ObjectType.slots.DelAttr = &delAttrSlot{objectDelAttr} ObjectType.slots.GetAttribute = &getAttributeSlot{objectGetAttribute} ObjectType.slots.Hash = &unaryOpSlot{objectHash} ObjectType.slots.New = &newSlot{objectNew} ObjectType.slots.SetAttr = &setAttrSlot{objectSetAttr} } // objectReconstructor builds an object from a class, its basis type and its // state (e.g. its string or integer value). It is similar to the // copy_reg._reconstructor function in CPython. func objectReconstructor(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "_reconstructor", args, TypeType, TypeType, ObjectType); raised != nil { return nil, raised } t, basisType, state := toTypeUnsafe(args[0]), toTypeUnsafe(args[1]), args[2] newMethod, raised := GetAttr(f, basisType.ToObject(), NewStr("__new__"), nil) if raised != nil { return nil, raised } o, raised := newMethod.Call(f, Args{t.ToObject(), state}, nil) if raised != nil { return nil, raised } if basisType != ObjectType { initMethod, raised := GetAttr(f, basisType.ToObject(), NewStr("__init__"), None) if raised != nil { return nil, raised } if initMethod != None { if _, raised := initMethod.Call(f, Args{o, state}, nil); raised != nil { return nil, raised } } } return o, nil } func objectReduceCommon(f *Frame, args Args) (*Object, *BaseException) { // TODO: Support __getstate__ and __getnewargs__. o := args[0] t := o.Type() proto := 0 if len(args) > 1 { proto = toIntUnsafe(args[1]).Value() } var raised *BaseException if proto < 2 { basisType := basisTypes[t.basis] if basisType == t { // Basis types are handled elsewhere by the pickle and // copy frameworks. This matches behavior in // copy_reg._reduce_ex in CPython. return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("can't pickle %s objects", t.Name())) } state := None if basisType != ObjectType { // For subclasses of basis types having state (e.g. // integer values), the state is captured by creating // an instance of that basis type. if state, raised = basisType.Call(f, Args{o}, nil); raised != nil { return nil, raised } } newArgs := NewTuple3(t.ToObject(), basisType.ToObject(), state).ToObject() if d := o.Dict(); d != nil { return NewTuple3(objectReconstructorFunc, newArgs, d.ToObject()).ToObject(), nil } return NewTuple2(objectReconstructorFunc, newArgs).ToObject(), nil } newArgs := []*Object{t.ToObject()} getNewArgsMethod, raised := GetAttr(f, o, NewStr("__getnewargs__"), None) if raised != nil { return nil, raised } if getNewArgsMethod != None { extraNewArgs, raised := getNewArgsMethod.Call(f, nil, nil) if raised != nil { return nil, raised } if !extraNewArgs.isInstance(TupleType) { format := "__getnewargs__ should return a tuple, not '%s'" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, extraNewArgs.Type().Name())) } newArgs = append(newArgs, toTupleUnsafe(extraNewArgs).elems...) } dict := None if d := o.Dict(); d != nil { dict = d.ToObject() } // For proto >= 2 include list and dict items. listItems := None if o.isInstance(ListType) { if listItems, raised = Iter(f, o); raised != nil { return nil, raised } } dictItems := None if o.isInstance(DictType) { iterItems, raised := o.typ.mroLookup(f, NewStr("iteritems")) if raised != nil { return nil, raised } if iterItems != nil { if dictItems, raised = iterItems.Call(f, Args{o}, nil); raised != nil { return nil, raised } } } newFunc, raised := GetAttr(f, t.ToObject(), NewStr("__new__"), nil) if raised != nil { return nil, raised } return NewTuple5(newFunc, NewTuple(newArgs...).ToObject(), dict, listItems, dictItems).ToObject(), nil } func objectGetDict(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "_get_dict", args, ObjectType); raised != nil { return nil, raised } o := args[0] d := o.Dict() if d == nil { format := "'%s' object has no attribute '__dict__'" return nil, f.RaiseType(AttributeErrorType, fmt.Sprintf(format, o.typ.Name())) } return args[0].Dict().ToObject(), nil } func objectSetDict(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "_set_dict", args, ObjectType, DictType); raised != nil { return nil, raised } o := args[0] if o.Type() == ObjectType { format := "'%s' object has no attribute '__dict__'" return nil, f.RaiseType(AttributeErrorType, fmt.Sprintf(format, o.typ.Name())) } o.setDict(toDictUnsafe(args[1])) return None, nil } ================================================ FILE: runtime/object_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" "regexp" "testing" ) func TestObjectCall(t *testing.T) { arg0 := newObject(ObjectType) arg1 := newObject(ObjectType) args := wrapArgs(arg0, arg1) kwargs := wrapKWArgs("kwarg", newObject(ObjectType)) kwargsDict := kwargs.makeDict() fn := func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { kwargsOrNone := None if len(kwargs) > 0 { kwargsOrNone = kwargs.makeDict().ToObject() } return newTestTuple(NewTuple(args.makeCopy()...), kwargsOrNone).ToObject(), nil } foo := newBuiltinFunction("foo", fn).ToObject() typ := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__call__": newBuiltinFunction("__call__", fn).ToObject(), })) callable := newObject(typ) raisesFunc := newBuiltinFunction("bar", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(RuntimeErrorType, "bar") }).ToObject() cases := []struct { callable *Object invokeTestCase }{ {foo, invokeTestCase{args: args, kwargs: kwargs, want: newTestTuple(NewTuple(args...), kwargsDict).ToObject()}}, {foo, invokeTestCase{args: args, want: newTestTuple(NewTuple(args...).ToObject(), None).ToObject()}}, {foo, invokeTestCase{kwargs: kwargs, want: newTestTuple(NewTuple(), kwargsDict).ToObject()}}, {foo, invokeTestCase{want: newTestTuple(NewTuple(), None).ToObject()}}, {foo, invokeTestCase{args: wrapArgs(arg0), want: newTestTuple(NewTuple(arg0), None).ToObject()}}, {callable, invokeTestCase{args: args, kwargs: kwargs, want: newTestTuple(NewTuple(callable, arg0, arg1), kwargsDict).ToObject()}}, {newObject(ObjectType), invokeTestCase{wantExc: mustCreateException(TypeErrorType, "'object' object is not callable")}}, {raisesFunc, invokeTestCase{wantExc: mustCreateException(RuntimeErrorType, "bar")}}, } for _, cas := range cases { if err := runInvokeTestCase(cas.callable, &cas.invokeTestCase); err != "" { t.Error(err) } } } func TestNewObject(t *testing.T) { cases := []*Type{DictType, ObjectType, StrType, TypeType} for _, c := range cases { if o := newObject(c); o.Type() != c { t.Errorf("new object has type %q, want %q", o.Type().Name(), c.Name()) } } } func TestObjectString(t *testing.T) { typ := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__repr__": newBuiltinFunction("__repr__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(ExceptionType, "ruh roh") }).ToObject(), })) cases := []struct { o *Object wantPattern string }{ {newObject(ObjectType), `^$`}, {NewTuple(NewStr("foo").ToObject(), NewStr("bar").ToObject()).ToObject(), `^\('foo', 'bar'\)$`}, {ExceptionType.ToObject(), "^$"}, {NewStr("foo\nbar").ToObject(), `^'foo\\nbar'$`}, {newObject(typ), `^$`}, } for _, cas := range cases { re := regexp.MustCompile(cas.wantPattern) s := cas.o.String() if matched := re.MatchString(s); !matched { t.Errorf("%v.String() = %q, doesn't match pattern %q", cas.o, s, re) } } } func TestObjectDelAttr(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object, name *Str) (*Object, *BaseException) { if raised := DelAttr(f, o, name); raised != nil { return nil, raised } return GetAttr(f, o, name, None) }) dellerType := newTestClass("Deller", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__get__": newBuiltinFunction("__get__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { attr, raised := args[1].Dict().GetItemString(f, "attr") if raised != nil { return nil, raised } if attr == nil { return nil, f.RaiseType(AttributeErrorType, "attr") } return attr, nil }).ToObject(), "__delete__": newBuiltinFunction("__delete__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { deleted, raised := args[1].Dict().DelItemString(f, "attr") if raised != nil { return nil, raised } if !deleted { return nil, f.RaiseType(AttributeErrorType, "attr") } return None, nil }).ToObject(), })) fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{"deller": newObject(dellerType)})) foo := newObject(fooType) if raised := foo.Dict().SetItemString(NewRootFrame(), "attr", NewInt(123).ToObject()); raised != nil { t.Fatal(raised) } cases := []invokeTestCase{ {args: wrapArgs(foo, "deller"), want: None}, {args: wrapArgs(newObject(fooType), "foo"), wantExc: mustCreateException(AttributeErrorType, "'Foo' object has no attribute 'foo'")}, {args: wrapArgs(newObject(fooType), "deller"), wantExc: mustCreateException(AttributeErrorType, "attr")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestObjectGetAttribute(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object, name *Str) (*Object, *BaseException) { return GetAttr(f, o, name, nil) }) // class Getter(object): // def __get__(self, *args): // return "got getter" getterType := newTestClass("Getter", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__get__": newBuiltinFunction("__get__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("got getter").ToObject(), nil }).ToObject(), })) getter := newObject(getterType) // class Setter(object): // def __get__(self, *args): // return "got setter" // def __set__(self, *args): // pass setterType := newTestClass("Setter", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__get__": newBuiltinFunction("__get__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("got setter").ToObject(), nil }).ToObject(), "__set__": newBuiltinFunction("__set__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return None, nil }).ToObject(), })) setter := newObject(setterType) // class Foo(object): // pass // foo = Foo() fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "bar": NewInt(42).ToObject(), "baz": NewStr("Foo's baz").ToObject(), "foogetter": getter, "foo": NewInt(101).ToObject(), "barsetter": setter, })) foo := newObject(fooType) if raised := foo.Dict().SetItemString(NewRootFrame(), "fooattr", True.ToObject()); raised != nil { t.Fatal(raised) } if raised := foo.Dict().SetItemString(NewRootFrame(), "barattr", NewInt(-1).ToObject()); raised != nil { t.Fatal(raised) } if raised := foo.Dict().SetItemString(NewRootFrame(), "barsetter", NewStr("NOT setter").ToObject()); raised != nil { t.Fatal(raised) } cases := []invokeTestCase{ {args: wrapArgs(foo, "bar"), want: NewInt(42).ToObject()}, {args: wrapArgs(foo, "fooattr"), want: True.ToObject()}, {args: wrapArgs(foo, "foogetter"), want: NewStr("got getter").ToObject()}, {args: wrapArgs(foo, "bar"), want: NewInt(42).ToObject()}, {args: wrapArgs(foo, "foo"), want: NewInt(101).ToObject()}, {args: wrapArgs(foo, "barattr"), want: NewInt(-1).ToObject()}, {args: wrapArgs(foo, "barsetter"), want: NewStr("got setter").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestObjectGetDict(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, NewDict()) foo := newObject(fooType) if raised := SetAttr(NewRootFrame(), foo, NewStr("bar"), NewInt(123).ToObject()); raised != nil { panic(raised) } fun := wrapFuncForTest(func(f *Frame, o *Object) (*Object, *BaseException) { return GetAttr(f, o, NewStr("__dict__"), nil) }) cases := []invokeTestCase{ {args: wrapArgs(newObject(ObjectType)), wantExc: mustCreateException(AttributeErrorType, "'object' object has no attribute '__dict__'")}, {args: wrapArgs(newObject(fooType)), want: NewDict().ToObject()}, {args: wrapArgs(foo), want: newStringDict(map[string]*Object{"bar": NewInt(123).ToObject()}).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestObjectSetDict(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, NewDict()) testDict := newStringDict(map[string]*Object{"bar": NewInt(123).ToObject()}) fun := wrapFuncForTest(func(f *Frame, o, val *Object) (*Object, *BaseException) { if raised := SetAttr(f, o, NewStr("__dict__"), val); raised != nil { return nil, raised } d := o.Dict() if d == nil { return None, nil } return d.ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(newObject(ObjectType), NewDict()), wantExc: mustCreateException(AttributeErrorType, "'object' object has no attribute '__dict__'")}, {args: wrapArgs(newObject(fooType), testDict), want: testDict.ToObject()}, {args: wrapArgs(newObject(fooType), 123), wantExc: mustCreateException(TypeErrorType, "'_set_dict' requires a 'dict' object but received a 'int'")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestObjectNew(t *testing.T) { foo := makeTestType("Foo", ObjectType) foo.flags &= ^typeFlagInstantiable prepareType(foo) cases := []invokeTestCase{ {args: wrapArgs(ExceptionType), want: newObject(ExceptionType)}, {args: wrapArgs(IntType), want: NewInt(0).ToObject()}, {wantExc: mustCreateException(TypeErrorType, "'__new__' requires 1 arguments")}, {args: wrapArgs(None), wantExc: mustCreateException(TypeErrorType, `'__new__' requires a 'type' object but received a "NoneType"`)}, {args: wrapArgs(foo), wantExc: mustCreateException(TypeErrorType, "object.__new__(Foo) is not safe, use Foo.__new__()")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(ObjectType, "__new__", &cas); err != "" { t.Error(err) } } } func TestObjectReduce(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, method *Str, o *Object, args Args) (*Object, *BaseException) { // Call __reduce/reduce_ex__. reduce, raised := GetAttr(f, o, method, nil) if raised != nil { return nil, raised } result, raised := reduce.Call(f, args, nil) if raised != nil { return nil, raised } msg := NewStr(fmt.Sprintf("reduce must return a tuple, got %s", result.Type().Name())).ToObject() if raised := Assert(f, GetBool(result.isInstance(TupleType)).ToObject(), msg); raised != nil { return nil, raised } elems := toTupleUnsafe(result).elems numElems := len(elems) msg = NewStr(fmt.Sprintf("reduce must return a tuple with 2 <= len <= 5, got %d", numElems)).ToObject() if raised := Assert(f, GetBool(numElems >= 2 && numElems <= 5).ToObject(), msg); raised != nil { return nil, raised } newArgs := elems[1] msg = NewStr(fmt.Sprintf("reduce second return value must be tuple, got %s", newArgs.Type().Name())).ToObject() if raised := Assert(f, GetBool(newArgs.isInstance(TupleType)).ToObject(), msg); raised != nil { return nil, raised } // Call the reconstructor function with the args returned. reduced, raised := elems[0].Call(f, toTupleUnsafe(newArgs).elems, nil) if raised != nil { return nil, raised } // Return the reconstructed object, object state, list items // and dict items. state, list, dict := None, None, None if numElems > 2 && elems[2] != None { state = elems[2] } if numElems > 3 && elems[3] != None { if list, raised = ListType.Call(f, Args{elems[3]}, nil); raised != nil { return nil, raised } } if numElems > 4 && elems[4] != None { if dict, raised = DictType.Call(f, Args{elems[4]}, nil); raised != nil { return nil, raised } } return NewTuple(reduced, state, list, dict).ToObject(), nil }) fooType := newTestClass("Foo", []*Type{StrType}, NewDict()) fooNoDict := &Str{Object: Object{typ: fooType}, value: "fooNoDict"} // Calling __reduce_ex__ on a type that overrides __reduce__ should // forward to the call to __reduce__. reduceOverrideType := newTestClass("ReduceOverride", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__reduce__": newBuiltinFunction("__reduce__", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { strNew, raised := GetAttr(f, StrType.ToObject(), NewStr("__new__"), nil) if raised != nil { return nil, raised } return newTestTuple(strNew, newTestTuple(StrType, "ReduceOverride")).ToObject(), nil }).ToObject(), })) getNewArgsRaisesType := newTestClass("GetNewArgsRaises", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__getnewargs__": newBuiltinFunction("__getnewargs__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(RuntimeErrorType, "uh oh") }).ToObject(), })) getNewArgsReturnsNonTupleType := newTestClass("GetNewArgsReturnsNonTuple", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__getnewargs__": newBuiltinFunction("__getnewargs__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(123).ToObject(), nil }).ToObject(), })) // Attempting to reduce an int will fail with "can't pickle" but // subclasses can be reduced. intSubclass := newTestClass("IntSubclass", []*Type{IntType}, NewDict()) intSubclassInst := &Int{Object{typ: intSubclass}, 123} cases := []invokeTestCase{ {args: wrapArgs("__reduce__", 42, Args{}), wantExc: mustCreateException(TypeErrorType, "can't pickle int objects")}, {args: wrapArgs("__reduce__", 42, wrapArgs(2)), want: newTestTuple(42, None, None, None).ToObject()}, {args: wrapArgs("__reduce_ex__", 42, Args{}), wantExc: mustCreateException(TypeErrorType, "can't pickle int objects")}, {args: wrapArgs("__reduce__", 3.14, wrapArgs("bad proto")), wantExc: mustCreateException(TypeErrorType, "'__reduce__' requires a 'int' object but received a 'str'")}, {args: wrapArgs("__reduce_ex__", 3.14, wrapArgs("bad proto")), wantExc: mustCreateException(TypeErrorType, "'__reduce_ex__' requires a 'int' object but received a 'str'")}, {args: wrapArgs("__reduce__", newObject(fooType), Args{}), want: newTestTuple("", NewDict(), None, None).ToObject()}, {args: wrapArgs("__reduce__", newObject(fooType), wrapArgs(2)), want: newTestTuple("", NewDict(), None, None).ToObject()}, {args: wrapArgs("__reduce_ex__", newObject(fooType), Args{}), want: newTestTuple("", NewDict(), None, None).ToObject()}, {args: wrapArgs("__reduce_ex__", newObject(reduceOverrideType), Args{}), want: newTestTuple("ReduceOverride", None, None, None).ToObject()}, {args: wrapArgs("__reduce__", fooNoDict, Args{}), want: newTestTuple("fooNoDict", None, None, None).ToObject()}, {args: wrapArgs("__reduce__", newTestList(1, 2, 3), wrapArgs(2)), want: newTestTuple(NewList(), None, newTestList(1, 2, 3), None).ToObject()}, {args: wrapArgs("__reduce__", newTestDict("a", 1, "b", 2), wrapArgs(2)), want: newTestTuple(NewDict(), None, None, newTestDict("a", 1, "b", 2)).ToObject()}, {args: wrapArgs("__reduce__", newObject(getNewArgsRaisesType), wrapArgs(2)), wantExc: mustCreateException(RuntimeErrorType, "uh oh")}, {args: wrapArgs("__reduce__", newObject(getNewArgsReturnsNonTupleType), wrapArgs(2)), wantExc: mustCreateException(TypeErrorType, "__getnewargs__ should return a tuple, not 'int'")}, {args: wrapArgs("__reduce__", newTestTuple("foo", "bar"), wrapArgs(2)), want: newTestTuple(newTestTuple("foo", "bar"), None, None, None).ToObject()}, {args: wrapArgs("__reduce__", 3.14, wrapArgs(2)), want: newTestTuple(3.14, None, None, None).ToObject()}, {args: wrapArgs("__reduce__", NewUnicode("abc"), wrapArgs(2)), want: newTestTuple(NewUnicode("abc"), None, None, None).ToObject()}, {args: wrapArgs("__reduce__", intSubclassInst, Args{}), want: newTestTuple(intSubclassInst, None, None, None).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestObjectSetAttr(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object, name *Str, value *Object) (*Object, *BaseException) { if raised := SetAttr(f, o, name, value); raised != nil { return nil, raised } return GetAttr(f, o, name, None) }) setterType := newTestClass("Setter", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__get__": newBuiltinFunction("__get__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { item, raised := args[1].Dict().GetItemString(f, "attr") if raised != nil { return nil, raised } if item == nil { return nil, raiseKeyError(f, NewStr("attr").ToObject()) } return item, nil }).ToObject(), "__set__": newBuiltinFunction("__set__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := args[1].Dict().SetItemString(f, "attr", NewTuple(args.makeCopy()...).ToObject()); raised != nil { return nil, raised } return None, nil }).ToObject(), })) setter := newObject(setterType) fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{"setter": setter})) foo := newObject(fooType) cases := []invokeTestCase{ {args: wrapArgs(newObject(fooType), "foo", "abc"), want: NewStr("abc").ToObject()}, {args: wrapArgs(foo, "setter", "baz"), want: NewTuple(setter, foo, NewStr("baz").ToObject()).ToObject()}, {args: wrapArgs(newObject(ObjectType), "foo", 10), wantExc: mustCreateException(AttributeErrorType, "'object' has no attribute 'foo'")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestObjectStrRepr(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object, wantPattern string) *BaseException { re := regexp.MustCompile(wantPattern) s, raised := ToStr(f, o) if raised != nil { return raised } if !re.MatchString(s.Value()) { t.Errorf("str(%v) = %v, want %q", o, s, re) } s, raised = Repr(f, o) if raised != nil { return raised } if !re.MatchString(s.Value()) { t.Errorf("repr(%v) = %v, want %q", o, s, re) } return nil }) type noReprMethodBasis struct{ Object } noReprMethodType := newType(TypeType, "noReprMethod", reflect.TypeOf(noReprMethodBasis{}), []*Type{}, NewDict()) noReprMethodType.mro = []*Type{noReprMethodType} fooType := newTestClass("Foo", []*Type{ObjectType}, newTestDict("__module__", "foo.bar")) cases := []invokeTestCase{ {args: wrapArgs(newObject(ObjectType), `^$`), want: None}, {args: wrapArgs(newObject(noReprMethodType), `^$`), want: None}, {args: wrapArgs(newObject(fooType), ``), want: None}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/param.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" ) // Param describes a parameter to a Python function. type Param struct { // Name is the argument name. Name string // Def is the default value to use if the argument is not provided. If // no default is specified then Def is nil. Def *Object } // ParamSpec describes a Python function's parameters. type ParamSpec struct { Count int name string minArgs int varArgIndex int kwArgIndex int params []Param } // NewParamSpec returns a new ParamSpec that accepts the given positional // parameters and optional vararg and/or kwarg parameter. func NewParamSpec(name string, params []Param, varArg bool, kwArg bool) *ParamSpec { s := &ParamSpec{name: name, params: params, varArgIndex: -1, kwArgIndex: -1} numParams := len(params) for ; s.minArgs < numParams; s.minArgs++ { if params[s.minArgs].Def != nil { break } } for _, p := range params[s.minArgs:numParams] { if p.Def == nil { format := "%s() non-keyword arg %s after keyword arg" logFatal(fmt.Sprintf(format, name, p.Name)) } } s.Count = numParams if varArg { s.varArgIndex = s.Count s.Count++ } if kwArg { s.kwArgIndex = s.Count s.Count++ } return s } // Validate ensures that a the args and kwargs passed are valid arguments for // the param spec s. The validated parameters are output to the validated slice // which must have len s.Count. func (s *ParamSpec) Validate(f *Frame, validated []*Object, args Args, kwargs KWArgs) *BaseException { if nv := len(validated); nv != s.Count { format := "%s(): validated slice was incorrect size: %d, want %d" return f.RaiseType(SystemErrorType, fmt.Sprintf(format, s.name, nv, s.Count)) } numParams := len(s.params) argc := len(args) if argc > numParams && s.varArgIndex == -1 { format := "%s() takes %d arguments (%d given)" return f.RaiseType(TypeErrorType, fmt.Sprintf(format, s.name, numParams, argc)) } i := 0 for ; i < argc && i < numParams; i++ { validated[i] = args[i] } if s.varArgIndex != -1 { validated[s.varArgIndex] = NewTuple(args[i:].makeCopy()...).ToObject() } var kwargDict *Dict if s.kwArgIndex != -1 { kwargDict = NewDict() validated[s.kwArgIndex] = kwargDict.ToObject() } for _, kw := range kwargs { name := kw.Name j := 0 for ; j < numParams; j++ { if s.params[j].Name == name { if validated[j] != nil { format := "%s() got multiple values for keyword argument '%s'" return f.RaiseType(TypeErrorType, fmt.Sprintf(format, s.name, name)) } validated[j] = kw.Value break } } if j == numParams { if kwargDict == nil { format := "%s() got an unexpected keyword argument '%s'" return f.RaiseType(TypeErrorType, fmt.Sprintf(format, s.name, name)) } if raised := kwargDict.SetItemString(f, name, kw.Value); raised != nil { return raised } } } for ; i < numParams; i++ { p := s.params[i] if validated[i] == nil { if p.Def == nil { format := "%s() takes at least %d arguments (%d given)" return f.RaiseType(TypeErrorType, fmt.Sprintf(format, s.name, s.minArgs, argc)) } validated[i] = p.Def } } return nil } ================================================ FILE: runtime/param_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestParamSpecValidate(t *testing.T) { testFunc := newBuiltinFunction("TestParamSpecValidate", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if len(args) < 1 { return nil, f.RaiseType(TypeErrorType, "not enough args") } val, raised := ToNative(f, args[0]) if raised != nil { return nil, raised } s, ok := val.Interface().(*ParamSpec) if !ok { return nil, f.RaiseType(TypeErrorType, "expected ParamSpec arg") } validated := make([]*Object, s.Count) if raised := s.Validate(f, validated, args[1:], kwargs); raised != nil { return nil, raised } return NewTuple(validated...).ToObject(), nil }) cases := []invokeTestCase{ invokeTestCase{args: wrapArgs(NewParamSpec("f1", nil, false, false)), want: NewTuple().ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f2", []Param{{"a", nil}}, false, false), 123), want: newTestTuple(123).ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f2", []Param{{"a", nil}}, false, false)), kwargs: wrapKWArgs("a", "apple"), want: newTestTuple("apple").ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f2", []Param{{"a", nil}}, false, false)), kwargs: wrapKWArgs("b", "bear"), wantExc: mustCreateException(TypeErrorType, "f2() got an unexpected keyword argument 'b'")}, invokeTestCase{args: wrapArgs(NewParamSpec("f2", []Param{{"a", nil}}, false, false)), wantExc: mustCreateException(TypeErrorType, "f2() takes at least 1 arguments (0 given)")}, invokeTestCase{args: wrapArgs(NewParamSpec("f2", []Param{{"a", nil}}, false, false), 1, 2, 3), wantExc: mustCreateException(TypeErrorType, "f2() takes 1 arguments (3 given)")}, invokeTestCase{args: wrapArgs(NewParamSpec("f3", []Param{{"a", nil}, {"b", nil}}, false, false), 1, 2), want: newTestTuple(1, 2).ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f3", []Param{{"a", nil}, {"b", nil}}, false, false), 1), kwargs: wrapKWArgs("b", "bear"), want: newTestTuple(1, "bear").ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f3", []Param{{"a", nil}, {"b", nil}}, false, false)), kwargs: wrapKWArgs("b", "bear", "a", "apple"), want: newTestTuple("apple", "bear").ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f3", []Param{{"a", nil}, {"b", nil}}, false, false), 1), kwargs: wrapKWArgs("a", "alpha"), wantExc: mustCreateException(TypeErrorType, "f3() got multiple values for keyword argument 'a'")}, invokeTestCase{args: wrapArgs(NewParamSpec("f4", []Param{{"a", nil}, {"b", None}}, false, false), 123), want: newTestTuple(123, None).ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f4", []Param{{"a", nil}, {"b", None}}, false, false), 123, "bar"), want: newTestTuple(123, "bar").ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f4", []Param{{"a", nil}, {"b", None}}, false, false)), kwargs: wrapKWArgs("a", 123, "b", "bar"), want: newTestTuple(123, "bar").ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f5", []Param{{"a", nil}}, true, false), 1), want: newTestTuple(1, NewTuple()).ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f5", []Param{{"a", nil}}, true, false), 1, 2, 3), want: newTestTuple(1, newTestTuple(2, 3)).ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f6", []Param{{"a", nil}}, false, true), "bar"), want: newTestTuple("bar", NewDict()).ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f6", []Param{{"a", nil}}, false, true)), kwargs: wrapKWArgs("a", "apple", "b", "bear"), want: newTestTuple("apple", newTestDict("b", "bear")).ToObject()}, invokeTestCase{args: wrapArgs(NewParamSpec("f6", []Param{{"a", nil}}, false, true), "bar"), kwargs: wrapKWArgs("b", "baz", "c", "qux"), want: newTestTuple("bar", newTestDict("b", "baz", "c", "qux")).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(testFunc.ToObject(), &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/range.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" "sync" ) var ( // enumerateType is the object representing the Python 'enumerate' type. enumerateType = newBasisType("enumerate", reflect.TypeOf(enumerate{}), toEnumerateUnsafe, ObjectType) // rangeIteratorType is the object representing the Python 'rangeiterator' type. rangeIteratorType = newBasisType("rangeiterator", reflect.TypeOf(rangeIterator{}), toRangeIteratorUnsafe, ObjectType) // xrangeType is the object representing the Python 'xrange' type. xrangeType = newBasisType("xrange", reflect.TypeOf(xrange{}), toXRangeUnsafe, ObjectType) ) type enumerate struct { Object mutex sync.Mutex index int iter *Object } func toEnumerateUnsafe(o *Object) *enumerate { return (*enumerate)(o.toPointer()) } func enumerateIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func enumerateNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{ObjectType, ObjectType} argc := len(args) if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkFunctionArgs(f, "__new__", args, expectedTypes...); raised != nil { return nil, raised } index := 0 if argc > 1 { if args[1].typ.slots.Index == nil { format := "%s object cannot be interpreted as an index" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, args[1].typ.Name())) } // TODO: support long? i, raised := IndexInt(f, args[1]) if raised != nil { return nil, raised } if i > 0 { index = i } } iter, raised := Iter(f, args[0]) if raised != nil { return nil, raised } for i := 0; i < index; i++ { _, raised := Next(f, iter) if raised != nil { if !raised.isInstance(StopIterationType) { return nil, raised } index = -1 f.RestoreExc(nil, nil) break } } var d *Dict if t != enumerateType { d = NewDict() } e := &enumerate{Object: Object{typ: t, dict: d}, index: index, iter: iter} return &e.Object, nil } func enumerateNext(f *Frame, o *Object) (ret *Object, raised *BaseException) { e := toEnumerateUnsafe(o) e.mutex.Lock() var item *Object if e.index != -1 { item, raised = Next(f, e.iter) } if raised == nil { if item == nil { raised = f.Raise(StopIterationType.ToObject(), nil, nil) e.index = -1 } else { ret = NewTuple2(NewInt(e.index).ToObject(), item).ToObject() e.index++ } } e.mutex.Unlock() return ret, raised } func initEnumerateType(map[string]*Object) { enumerateType.slots.Iter = &unaryOpSlot{enumerateIter} enumerateType.slots.Next = &unaryOpSlot{enumerateNext} enumerateType.slots.New = &newSlot{enumerateNew} } // TODO: Synchronize access to this structure. type rangeIterator struct { Object i int stop int step int } func toRangeIteratorUnsafe(o *Object) *rangeIterator { return (*rangeIterator)(o.toPointer()) } func rangeIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func rangeIteratorNext(f *Frame, o *Object) (*Object, *BaseException) { iter := toRangeIteratorUnsafe(o) if iter.i == iter.stop { return nil, f.Raise(StopIterationType.ToObject(), nil, nil) } ret := NewInt(iter.i) iter.i += iter.step return ret.ToObject(), nil } func initRangeIteratorType(map[string]*Object) { rangeIteratorType.flags &^= typeFlagInstantiable | typeFlagBasetype rangeIteratorType.slots.Iter = &unaryOpSlot{rangeIteratorIter} rangeIteratorType.slots.Next = &unaryOpSlot{rangeIteratorNext} } type xrange struct { Object start int stop int step int } func toXRangeUnsafe(o *Object) *xrange { return (*xrange)(o.toPointer()) } func xrangeGetItem(f *Frame, o, key *Object) (*Object, *BaseException) { if key.typ.slots.Index == nil { format := "sequence index must be integer, not '%s'" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, key.typ.Name())) } i, raised := IndexInt(f, key) if raised != nil { return nil, raised } r := toXRangeUnsafe(o) i, raised = seqCheckedIndex(f, (r.stop-r.start)/r.step, i) if raised != nil { return nil, raised } return NewInt(r.start + i*r.step).ToObject(), nil } func xrangeIter(f *Frame, o *Object) (*Object, *BaseException) { r := toXRangeUnsafe(o) return &(&rangeIterator{Object{typ: rangeIteratorType}, r.start, r.stop, r.step}).Object, nil } func xrangeLen(f *Frame, o *Object) (*Object, *BaseException) { r := toXRangeUnsafe(o) return NewInt((r.stop - r.start) / r.step).ToObject(), nil } func xrangeNew(f *Frame, _ *Type, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{IntType, IntType, IntType} argc := len(args) if argc > 0 && argc < 3 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "__new__", args, expectedTypes...); raised != nil { return nil, raised } start, stop, step := 0, 0, 1 if argc == 1 { stop = toIntUnsafe(args[0]).Value() } else { start = toIntUnsafe(args[0]).Value() stop = toIntUnsafe(args[1]).Value() if argc > 2 { step = toIntUnsafe(args[2]).Value() } } stop, _, result := seqRange(start, stop, step) switch result { case seqRangeZeroStep: return nil, f.RaiseType(ValueErrorType, "xrange() arg 3 must not be zero") case seqRangeOverflow: return nil, f.RaiseType(OverflowErrorType, errResultTooLarge) } r := &xrange{Object: Object{typ: xrangeType}, start: start, stop: stop, step: step} return &r.Object, nil } func xrangeRepr(_ *Frame, o *Object) (*Object, *BaseException) { r := toXRangeUnsafe(o) s := "" if r.step != 1 { s = fmt.Sprintf("xrange(%d, %d, %d)", r.start, r.stop, r.step) } else if r.start != 0 { s = fmt.Sprintf("xrange(%d, %d)", r.start, r.stop) } else { s = fmt.Sprintf("xrange(%d)", r.stop) } return NewStr(s).ToObject(), nil } func initXRangeType(map[string]*Object) { xrangeType.flags &^= typeFlagBasetype xrangeType.slots.GetItem = &binaryOpSlot{xrangeGetItem} xrangeType.slots.Iter = &unaryOpSlot{xrangeIter} xrangeType.slots.Len = &unaryOpSlot{xrangeLen} xrangeType.slots.New = &newSlot{xrangeNew} xrangeType.slots.Repr = &unaryOpSlot{xrangeRepr} } ================================================ FILE: runtime/range_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestEnumerate(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, args ...*Object) (*Object, *BaseException) { e, raised := enumerateType.Call(f, args, nil) if raised != nil { return nil, raised } return ListType.Call(f, Args{e}, nil) }) cases := []invokeTestCase{ {args: wrapArgs(NewTuple()), want: NewList().ToObject()}, {args: wrapArgs(newTestList("foo", "bar")), want: newTestList(newTestTuple(0, "foo"), newTestTuple(1, "bar")).ToObject()}, {args: wrapArgs(newTestTuple("foo", "bar"), 1), want: newTestList(newTestTuple(1, "bar")).ToObject()}, {args: wrapArgs(newTestList("foo", "bar"), 128), want: NewList().ToObject()}, {args: wrapArgs(newTestTuple(42), -300), want: newTestList(newTestTuple(0, 42)).ToObject()}, {args: wrapArgs(NewTuple(), 3.14), wantExc: mustCreateException(TypeErrorType, "float object cannot be interpreted as an index")}, {args: wrapArgs(123), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {args: wrapArgs(), wantExc: mustCreateException(TypeErrorType, "'__new__' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestRangeIteratorIter(t *testing.T) { fun := wrapFuncForTest(func(f *Frame) *BaseException { xrange, raised := xrangeType.Call(f, wrapArgs(5), nil) if raised != nil { return raised } iter, raised := Iter(f, xrange) if raised != nil { return raised } if !iter.isInstance(rangeIteratorType) { t.Errorf("iter(xrange(5)) = %v, want rangeiterator", iter) } got, raised := Iter(f, iter) if raised != nil { return raised } if got != iter { t.Errorf("iter(%[1]v) = %[2]v, want %[1]v", iter, got) } return nil }) if err := runInvokeTestCase(fun, &invokeTestCase{want: None}); err != "" { t.Error(err) } } func TestXRangeGetItem(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestXRange(10), 3), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestXRange(10, 12), 1), want: NewInt(11).ToObject()}, {args: wrapArgs(newTestXRange(5, -2, -3), 2), want: NewInt(-1).ToObject()}, {args: wrapArgs(newTestXRange(3), 100), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(newTestXRange(5), newTestSlice(1, 3)), wantExc: mustCreateException(TypeErrorType, "sequence index must be integer, not 'slice'")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(xrangeType, "__getitem__", &cas); err != "" { t.Error(err) } } } func TestXRangeLen(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestXRange(10)), want: NewInt(10).ToObject()}, {args: wrapArgs(newTestXRange(10, 12)), want: NewInt(2).ToObject()}, {args: wrapArgs(newTestXRange(5, 16, 5)), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestXRange(5, -2, -3)), want: NewInt(3).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(xrangeType, "__len__", &cas); err != "" { t.Error(err) } } } func TestXRangeNew(t *testing.T) { fun := newBuiltinFunction("TestXRangeNew", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { xrange, raised := xrangeType.Call(f, args, nil) if raised != nil { return nil, raised } return ListType.Call(f, []*Object{xrange}, nil) }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(5), want: newTestList(0, 1, 2, 3, 4).ToObject()}, {args: wrapArgs(-3, 1), want: newTestList(-3, -2, -1, 0).ToObject()}, {args: wrapArgs(10, 0), want: NewList().ToObject()}, {args: wrapArgs(4, 7, 3), want: newTestList(4).ToObject()}, {args: wrapArgs(4, 8, 3), want: newTestList(4, 7).ToObject()}, {args: wrapArgs(-12, -21, -5), want: newTestList(-12, -17).ToObject()}, {args: wrapArgs(-12, -22, -5), want: newTestList(-12, -17).ToObject()}, {args: wrapArgs(-12, -23, -5), want: newTestList(-12, -17, -22).ToObject()}, {args: wrapArgs(4, -4), want: NewList().ToObject()}, {args: wrapArgs(-26, MinInt), want: NewList().ToObject()}, {args: wrapArgs(1, 2, 0), wantExc: mustCreateException(ValueErrorType, "xrange() arg 3 must not be zero")}, {args: wrapArgs(0, MinInt, -1), wantExc: mustCreateException(OverflowErrorType, "result too large")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestXRangeRepr(t *testing.T) { fun := newBuiltinFunction("TestXRangeRepr", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { xrange, raised := xrangeType.Call(f, args, nil) if raised != nil { return nil, raised } s, raised := Repr(f, xrange) if raised != nil { return nil, raised } return s.ToObject(), nil }).ToObject() cases := []invokeTestCase{ {args: wrapArgs(42), want: NewStr("xrange(42)").ToObject()}, {args: wrapArgs(42, 48), want: NewStr("xrange(42, 48)").ToObject()}, {args: wrapArgs(42, 10), want: NewStr("xrange(42, 42)").ToObject()}, {args: wrapArgs(-10, 10), want: NewStr("xrange(-10, 10)").ToObject()}, {args: wrapArgs(-10, 10, 10), want: NewStr("xrange(-10, 10, 10)").ToObject()}, {args: wrapArgs(-10, 10, 3), want: NewStr("xrange(-10, 11, 3)").ToObject()}, {args: wrapArgs(4, 7, 3), want: NewStr("xrange(4, 7, 3)").ToObject()}, {args: wrapArgs(4, 8, 3), want: NewStr("xrange(4, 10, 3)").ToObject()}, {args: wrapArgs(-10, 10, -3), want: NewStr("xrange(-10, -10, -3)").ToObject()}, {args: wrapArgs(3, 3, -5), want: NewStr("xrange(3, 3, -5)").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func newTestXRange(args ...interface{}) *Object { return mustNotRaise(xrangeType.Call(NewRootFrame(), wrapArgs(args...), nil)) } ================================================ FILE: runtime/seq.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy // This file contains common code and helpers for sequence types. import ( "bytes" "fmt" "reflect" "sync" ) var ( seqIteratorType = newBasisType("iterator", reflect.TypeOf(seqIterator{}), toSeqIteratorUnsafe, ObjectType) ) func seqAdd(f *Frame, elems1, elems2 []*Object) ([]*Object, *BaseException) { if len(elems1)+len(elems2) < 0 { // This indicates an int overflow. return nil, f.RaiseType(OverflowErrorType, errResultTooLarge) } return append(elems1, elems2...), nil } func seqCompare(f *Frame, elems1, elems2 []*Object, cmp binaryOpFunc) (*Object, *BaseException) { n1 := len(elems1) n2 := len(elems2) for i := 0; i < n1 && i < n2; i++ { eq, raised := Eq(f, elems1[i], elems2[i]) if raised != nil { return nil, raised } if ret, raised := IsTrue(f, eq); raised != nil { return nil, raised } else if !ret { // We encountered an unequal element before the end of // either sequence so perform the comparison on the two // elements. return cmp(f, elems1[i], elems2[i]) } } // One sequence is longer than the other, so do the comparison on the // lengths of the two sequences. return cmp(f, NewInt(n1).ToObject(), NewInt(n2).ToObject()) } // seqApply calls fun with a slice of objects contained in the sequence object // seq. If the second callback parameter is true, the slice is borrowed and the // function must not modify the provided slice. Otherwise the slice is scratch // and it may freely be used in any way. It will raise if seq is not a sequence // object. func seqApply(f *Frame, seq *Object, fun func([]*Object, bool) *BaseException) *BaseException { switch { // Don't use fast path referencing the underlying slice directly for // list and tuple subtypes. See comment in listextend in listobject.c. case seq.typ == ListType: l := toListUnsafe(seq) l.mutex.RLock() raised := fun(l.elems, true) l.mutex.RUnlock() return raised case seq.typ == TupleType: return fun(toTupleUnsafe(seq).elems, true) default: elems := []*Object{} raised := seqForEach(f, seq, func(elem *Object) *BaseException { elems = append(elems, elem) return nil }) if raised != nil { return raised } return fun(elems, false) } } func seqCheckedIndex(f *Frame, seqLen, index int) (int, *BaseException) { if index < 0 { index = seqLen + index } if index < 0 || index >= seqLen { return 0, f.RaiseType(IndexErrorType, "index out of range") } return index, nil } func seqClampIndex(i, seqLen int) int { if i < 0 { i += seqLen if i < 0 { i = 0 } } if i > seqLen { i = seqLen } return i } func seqContains(f *Frame, iterable *Object, v *Object) (*Object, *BaseException) { pred := func(o *Object) (bool, *BaseException) { eq, raised := Eq(f, v, o) if raised != nil { return false, raised } ret, raised := IsTrue(f, eq) if raised != nil { return false, raised } return ret, nil } foundEqItem, raised := seqFindFirst(f, iterable, pred) if raised != nil { return nil, raised } return GetBool(foundEqItem).ToObject(), raised } func seqCount(f *Frame, iterable *Object, v *Object) (*Object, *BaseException) { count := 0 raised := seqForEach(f, iterable, func(o *Object) *BaseException { eq, raised := Eq(f, o, v) if raised != nil { return raised } t, raised := IsTrue(f, eq) if raised != nil { return raised } if t { count++ } return nil }) if raised != nil { return nil, raised } return NewInt(count).ToObject(), nil } func seqFindFirst(f *Frame, iterable *Object, pred func(*Object) (bool, *BaseException)) (bool, *BaseException) { iter, raised := Iter(f, iterable) if raised != nil { return false, raised } item, raised := Next(f, iter) for ; raised == nil; item, raised = Next(f, iter) { ret, raised := pred(item) if raised != nil { return false, raised } if ret { return true, nil } } if !raised.isInstance(StopIterationType) { return false, raised } f.RestoreExc(nil, nil) return false, nil } func seqFindElem(f *Frame, elems []*Object, o *Object) (int, *BaseException) { for i, elem := range elems { eq, raised := Eq(f, elem, o) if raised != nil { return -1, raised } found, raised := IsTrue(f, eq) if raised != nil { return -1, raised } if found { return i, nil } } return -1, nil } func seqForEach(f *Frame, iterable *Object, callback func(*Object) *BaseException) *BaseException { iter, raised := Iter(f, iterable) if raised != nil { return raised } item, raised := Next(f, iter) for ; raised == nil; item, raised = Next(f, iter) { if raised := callback(item); raised != nil { return raised } } if !raised.isInstance(StopIterationType) { return raised } f.RestoreExc(nil, nil) return nil } // seqGetItem returns a single element or a slice of elements of elems // depending on whether index is an integer or a slice. If index is neither of // those types then a TypeError is returned. func seqGetItem(f *Frame, elems []*Object, index *Object) (*Object, []*Object, *BaseException) { switch { case index.typ.slots.Index != nil: i, raised := IndexInt(f, index) if raised != nil { return nil, nil, raised } i, raised = seqCheckedIndex(f, len(elems), i) if raised != nil { return nil, nil, raised } return elems[i], nil, nil case index.isInstance(SliceType): s := toSliceUnsafe(index) start, stop, step, sliceLen, raised := s.calcSlice(f, len(elems)) if raised != nil { return nil, nil, raised } result := make([]*Object, sliceLen) i := 0 for j := start; j != stop; j += step { result[i] = elems[j] i++ } return nil, result, nil } return nil, nil, f.RaiseType(TypeErrorType, fmt.Sprintf("sequence indices must be integers, not %s", index.typ.Name())) } func seqMul(f *Frame, elems []*Object, n int) ([]*Object, *BaseException) { if n <= 0 { return nil, nil } numElems := len(elems) if numElems > MaxInt/n { return nil, f.RaiseType(OverflowErrorType, errResultTooLarge) } newNumElems := numElems * n resultElems := make([]*Object, newNumElems) for i := 0; i < newNumElems; i++ { resultElems[i] = elems[i%numElems] } return resultElems, nil } func seqNew(f *Frame, args Args) ([]*Object, *BaseException) { if len(args) == 0 { return nil, nil } if raised := checkMethodArgs(f, "__new__", args, ObjectType); raised != nil { return nil, raised } var result []*Object raised := seqApply(f, args[0], func(elems []*Object, borrowed bool) *BaseException { if borrowed { result = make([]*Object, len(elems)) copy(result, elems) } else { result = elems } return nil }) if raised != nil { return nil, raised } return result, nil } type seqRangeResult int const ( seqRangeOK seqRangeResult = iota seqRangeOverflow seqRangeZeroStep ) // seqRange takes the bounds and stride defining a Python range (e.g. // xrange(start, stop, step)) and returns three things: // // 1. The terminal value for the range when iterating // 2. The length of the range (i.e. the number of iterations) // 3. A status indicating whether the range is valid // // The terminal value can be used to iterate over the range as follows: // // for i := start; i != term; i += step { ... } func seqRange(start, stop, step int) (int, int, seqRangeResult) { if step == 0 { return 0, 0, seqRangeZeroStep } if stop == start || (stop > start) != (step > 0) { // The step doesn't make progress towards the goal, // so return an empty range. return start, 0, seqRangeOK } if step > 0 { stop-- } else { stop++ } n := (stop-start)/step + 1 if n < 0 { return 0, 0, seqRangeOverflow } return start + n*step, n, seqRangeOK } func seqRepr(f *Frame, elems []*Object) (string, *BaseException) { var buf bytes.Buffer for i, o := range elems { if i > 0 { buf.WriteString(", ") } s, raised := Repr(f, o) if raised != nil { return "", raised } buf.WriteString(s.Value()) i++ } return buf.String(), nil } func seqWrapEach(f *Frame, elems ...interface{}) ([]*Object, *BaseException) { result := make([]*Object, len(elems)) for i, elem := range elems { var raised *BaseException if result[i], raised = WrapNative(f, reflect.ValueOf(elem)); raised != nil { return nil, raised } } return result, nil } type seqIterator struct { Object seq *Object mutex sync.Mutex index int } func newSeqIterator(seq *Object) *Object { iter := &seqIterator{Object: Object{typ: seqIteratorType}, seq: seq} return &iter.Object } func toSeqIteratorUnsafe(o *Object) *seqIterator { return (*seqIterator)(o.toPointer()) } func seqIteratorIter(f *Frame, o *Object) (*Object, *BaseException) { return o, nil } func seqIteratorNext(f *Frame, o *Object) (item *Object, raised *BaseException) { i := toSeqIteratorUnsafe(o) i.mutex.Lock() if i.seq == nil { raised = f.Raise(StopIterationType.ToObject(), nil, nil) } else if item, raised = GetItem(f, i.seq, NewInt(i.index).ToObject()); raised == nil { i.index++ } else if raised.isInstance(IndexErrorType) { i.seq = nil raised = f.Raise(StopIterationType.ToObject(), nil, nil) } i.mutex.Unlock() return item, raised } func initSeqIteratorType(map[string]*Object) { seqIteratorType.flags &= ^(typeFlagBasetype | typeFlagInstantiable) seqIteratorType.slots.Iter = &unaryOpSlot{seqIteratorIter} seqIteratorType.slots.Next = &unaryOpSlot{seqIteratorNext} } ================================================ FILE: runtime/seq_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestSeqApply(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, seq *Object) (*Object, *BaseException) { var got *Object raised := seqApply(f, seq, func(elems []*Object, borrowed bool) *BaseException { got = newTestTuple(NewTuple(elems...), GetBool(borrowed)).ToObject() return nil }) if raised != nil { return nil, raised } return got, nil }) cases := []invokeTestCase{ {args: wrapArgs(NewTuple()), want: newTestTuple(NewTuple(), true).ToObject()}, {args: wrapArgs(newTestList("foo", "bar")), want: newTestTuple(newTestTuple("foo", "bar"), true).ToObject()}, {args: wrapArgs(newTestDict("foo", None)), want: newTestTuple(newTestTuple("foo"), false).ToObject()}, {args: wrapArgs(42), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSeqCount(t *testing.T) { badEqType := newTestClass("Eq", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, "uh oh") }).ToObject(), })) badNonZeroType := newTestClass("BadNonZeroType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__nonzero__": newBuiltinFunction("__nonzero__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return nil, f.RaiseType(TypeErrorType, "uh oh") }).ToObject(), })) worseEqType := newTestClass("WorseEqCmp", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return newObject(badNonZeroType), nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(newTestList(), NewInt(1)), want: NewInt(0).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 4, 5, 6), NewInt(7)), want: NewInt(0).ToObject()}, {args: wrapArgs(newTestList(1, 2, 3, 2, 2, 4), NewInt(2)), want: NewInt(3).ToObject()}, {args: wrapArgs(newTestList(1, None, None, 3), None), want: NewInt(2).ToObject()}, {args: wrapArgs(newTestList("a", "b", "c", "d", "e"), NewStr("c")), want: NewInt(1).ToObject()}, {args: wrapArgs(newTestList(newObject(badEqType)), newObject(badEqType)), wantExc: mustCreateException(TypeErrorType, "uh oh")}, {args: wrapArgs(newTestList(newObject(worseEqType)), newObject(worseEqType)), wantExc: mustCreateException(TypeErrorType, "uh oh")}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(seqCount), &cas); err != "" { t.Error(err) } } } func TestSeqForEach(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, seq *Object) (*Object, *BaseException) { elems := []*Object{} raised := seqForEach(f, seq, func(elem *Object) *BaseException { elems = append(elems, elem) return nil }) if raised != nil { return nil, raised } return NewTuple(elems...).ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(NewList()), want: NewTuple().ToObject()}, {args: wrapArgs(newTestDict("foo", 1, "bar", 2)), want: newTestTuple("foo", "bar").ToObject()}, {args: wrapArgs(123), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSeqIterator(t *testing.T) { fun := newBuiltinFunction("TestSeqIterator", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return TupleType.Call(f, args, nil) }).ToObject() exhaustedIter := newSeqIterator(NewStr("foo").ToObject()) TupleType.Call(NewRootFrame(), []*Object{exhaustedIter}, nil) cases := []invokeTestCase{ {args: wrapArgs(newSeqIterator(NewStr("bar").ToObject())), want: newTestTuple("b", "a", "r").ToObject()}, {args: wrapArgs(newSeqIterator(newTestTuple(123, 456).ToObject())), want: newTestTuple(123, 456).ToObject()}, {args: wrapArgs(exhaustedIter), want: NewTuple().ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSeqNew(t *testing.T) { fun := newBuiltinFunction("TestSeqNew", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { elems, raised := seqNew(f, args) if raised != nil { return nil, raised } return NewTuple(elems...).ToObject(), nil }).ToObject() cases := []invokeTestCase{ {want: NewTuple().ToObject()}, {args: wrapArgs(newTestTuple("foo", "bar")), want: newTestTuple("foo", "bar").ToObject()}, {args: wrapArgs(newTestDict("foo", 1)), want: newTestTuple("foo").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/set.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" ) var ( // FrozenSetType is the object representing the Python 'set' type. FrozenSetType = newBasisType("frozenset", reflect.TypeOf(FrozenSet{}), toFrozenSetUnsafe, ObjectType) // SetType is the object representing the Python 'set' type. SetType = newBasisType("set", reflect.TypeOf(Set{}), toSetUnsafe, ObjectType) ) type setBase struct { Object dict *Dict } func (s *setBase) contains(f *Frame, key *Object) (bool, *BaseException) { item, raised := s.dict.GetItem(f, key) if raised != nil { return false, raised } return item != nil, nil } func (s *setBase) isSubset(f *Frame, o *Object) (*Object, *BaseException) { s2, raised := setFromSeq(f, o) if raised != nil { return nil, raised } return setCompare(f, compareOpLE, s, &s2.Object) } func (s *setBase) isSuperset(f *Frame, o *Object) (*Object, *BaseException) { s2, raised := setFromSeq(f, o) if raised != nil { return nil, raised } return setCompare(f, compareOpGE, s, &s2.Object) } func (s *setBase) repr(f *Frame) (*Object, *BaseException) { if f.reprEnter(&s.Object) { return NewStr(fmt.Sprintf("%s(...)", s.typ.Name())).ToObject(), nil } repr, raised := Repr(f, s.dict.Keys(f).ToObject()) f.reprLeave(&s.Object) if raised != nil { return nil, raised } return NewStr(fmt.Sprintf("%s(%s)", s.typ.Name(), repr.Value())).ToObject(), nil } // Set represents Python 'set' objects. type Set setBase // NewSet returns an empty Set. func NewSet() *Set { return &Set{Object{typ: SetType}, NewDict()} } func toSetUnsafe(o *Object) *Set { return (*Set)(o.toPointer()) } // Add inserts key into s. If key already exists then does nothing. func (s *Set) Add(f *Frame, key *Object) (bool, *BaseException) { origin, raised := s.dict.putItem(f, key, None, true) if raised != nil { return false, raised } return origin == nil, nil } // Contains returns true if key exists in s. func (s *Set) Contains(f *Frame, key *Object) (bool, *BaseException) { return (*setBase)(s).contains(f, key) } // Remove erases key from s. If key is not in s then raises KeyError. func (s *Set) Remove(f *Frame, key *Object) (bool, *BaseException) { return s.dict.DelItem(f, key) } // ToObject upcasts s to an Object. func (s *Set) ToObject() *Object { return &s.Object } // Update inserts all elements in the iterable o into s. func (s *Set) Update(f *Frame, o *Object) *BaseException { raised := seqForEach(f, o, func(key *Object) *BaseException { return s.dict.SetItem(f, key, None) }) return raised } func setAdd(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "add", args, SetType, ObjectType); raised != nil { return nil, raised } if _, raised := toSetUnsafe(args[0]).Add(f, args[1]); raised != nil { return nil, raised } return None, nil } func setContains(f *Frame, seq, value *Object) (*Object, *BaseException) { contains, raised := toSetUnsafe(seq).Contains(f, value) if raised != nil { return nil, raised } return GetBool(contains).ToObject(), nil } func setDiscard(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "discard", args, SetType, ObjectType); raised != nil { return nil, raised } if _, raised := toSetUnsafe(args[0]).Remove(f, args[1]); raised != nil { return nil, raised } return None, nil } func setEq(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpEq, (*setBase)(toSetUnsafe(v)), w) } func setGE(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpGE, (*setBase)(toSetUnsafe(v)), w) } func setGT(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpGT, (*setBase)(toSetUnsafe(v)), w) } func setInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) if argc > 1 { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("set expected at most 1 arguments, got %d", argc)) } s := toSetUnsafe(o) if argc == 1 { if raised := s.Update(f, args[0]); raised != nil { return nil, raised } } return None, nil } func setIsSubset(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "issubset", args, SetType, ObjectType); raised != nil { return nil, raised } return (*setBase)(toSetUnsafe(args[0])).isSubset(f, args[1]) } func setIsSuperset(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "issuperset", args, SetType, ObjectType); raised != nil { return nil, raised } return (*setBase)(toSetUnsafe(args[0])).isSuperset(f, args[1]) } func setIter(f *Frame, o *Object) (*Object, *BaseException) { s := toSetUnsafe(o) s.dict.mutex.Lock(f) iter := &newDictKeyIterator(s.dict).Object s.dict.mutex.Unlock(f) return iter, nil } func setLE(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpLE, (*setBase)(toSetUnsafe(v)), w) } func setLen(f *Frame, o *Object) (*Object, *BaseException) { return NewInt(toSetUnsafe(o).dict.Len()).ToObject(), nil } func setLT(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpLT, (*setBase)(toSetUnsafe(v)), w) } func setNE(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpNE, (*setBase)(toSetUnsafe(v)), w) } func setNew(f *Frame, t *Type, _ Args, _ KWArgs) (*Object, *BaseException) { s := toSetUnsafe(newObject(t)) s.dict = NewDict() return s.ToObject(), nil } func setRemove(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "remove", args, SetType, ObjectType); raised != nil { return nil, raised } key := args[1] if removed, raised := toSetUnsafe(args[0]).Remove(f, key); raised != nil { return nil, raised } else if !removed { return nil, raiseKeyError(f, key) } return None, nil } func setRepr(f *Frame, o *Object) (*Object, *BaseException) { return (*setBase)(toSetUnsafe(o)).repr(f) } func setUpdate(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "update", args, SetType, ObjectType); raised != nil { return nil, raised } if raised := toSetUnsafe(args[0]).Update(f, args[1]); raised != nil { return nil, raised } return None, nil } func initSetType(dict map[string]*Object) { dict["add"] = newBuiltinFunction("add", setAdd).ToObject() dict["discard"] = newBuiltinFunction("discard", setDiscard).ToObject() dict["issubset"] = newBuiltinFunction("issubset", setIsSubset).ToObject() dict["issuperset"] = newBuiltinFunction("issuperset", setIsSuperset).ToObject() dict["remove"] = newBuiltinFunction("remove", setRemove).ToObject() dict["update"] = newBuiltinFunction("update", setUpdate).ToObject() SetType.slots.Contains = &binaryOpSlot{setContains} SetType.slots.Eq = &binaryOpSlot{setEq} SetType.slots.GE = &binaryOpSlot{setGE} SetType.slots.GT = &binaryOpSlot{setGT} SetType.slots.Hash = &unaryOpSlot{hashNotImplemented} SetType.slots.Init = &initSlot{setInit} SetType.slots.Iter = &unaryOpSlot{setIter} SetType.slots.LE = &binaryOpSlot{setLE} SetType.slots.Len = &unaryOpSlot{setLen} SetType.slots.LT = &binaryOpSlot{setLT} SetType.slots.NE = &binaryOpSlot{setNE} SetType.slots.New = &newSlot{setNew} SetType.slots.Repr = &unaryOpSlot{setRepr} } // FrozenSet represents Python 'set' objects. type FrozenSet setBase func toFrozenSetUnsafe(o *Object) *FrozenSet { return (*FrozenSet)(o.toPointer()) } // Contains returns true if key exists in s. func (s *FrozenSet) Contains(f *Frame, key *Object) (bool, *BaseException) { return (*setBase)(s).contains(f, key) } // ToObject upcasts s to an Object. func (s *FrozenSet) ToObject() *Object { return &s.Object } func frozenSetContains(f *Frame, seq, value *Object) (*Object, *BaseException) { contains, raised := toFrozenSetUnsafe(seq).Contains(f, value) if raised != nil { return nil, raised } return GetBool(contains).ToObject(), nil } func frozenSetEq(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpEq, (*setBase)(toFrozenSetUnsafe(v)), w) } func frozenSetGE(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpGE, (*setBase)(toFrozenSetUnsafe(v)), w) } func frozenSetGT(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpGT, (*setBase)(toFrozenSetUnsafe(v)), w) } func frozenSetIsSubset(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "issubset", args, FrozenSetType, ObjectType); raised != nil { return nil, raised } return (*setBase)(toFrozenSetUnsafe(args[0])).isSubset(f, args[1]) } func frozenSetIsSuperset(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "issuperset", args, FrozenSetType, ObjectType); raised != nil { return nil, raised } return (*setBase)(toFrozenSetUnsafe(args[0])).isSuperset(f, args[1]) } func frozenSetIter(f *Frame, o *Object) (*Object, *BaseException) { s := toFrozenSetUnsafe(o) s.dict.mutex.Lock(f) iter := &newDictKeyIterator(s.dict).Object s.dict.mutex.Unlock(f) return iter, nil } func frozenSetLE(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpLE, (*setBase)(toFrozenSetUnsafe(v)), w) } func frozenSetLen(f *Frame, o *Object) (*Object, *BaseException) { return NewInt(toFrozenSetUnsafe(o).dict.Len()).ToObject(), nil } func frozenSetLT(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpLT, (*setBase)(toFrozenSetUnsafe(v)), w) } func frozenSetNE(f *Frame, v, w *Object) (*Object, *BaseException) { return setCompare(f, compareOpNE, (*setBase)(toFrozenSetUnsafe(v)), w) } func frozenSetNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { argc := len(args) if argc > 1 { format := "frozenset expected at most 1 arguments, got %d" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, argc)) } s := toFrozenSetUnsafe(newObject(t)) s.dict = NewDict() if argc == 1 { raised := seqForEach(f, args[0], func(o *Object) *BaseException { return s.dict.SetItem(f, o, None) }) if raised != nil { return nil, raised } } return s.ToObject(), nil } func frozenSetRepr(f *Frame, o *Object) (*Object, *BaseException) { return (*setBase)(toFrozenSetUnsafe(o)).repr(f) } func initFrozenSetType(dict map[string]*Object) { dict["issubset"] = newBuiltinFunction("issubset", frozenSetIsSubset).ToObject() dict["issuperset"] = newBuiltinFunction("issuperset", frozenSetIsSuperset).ToObject() FrozenSetType.slots.Contains = &binaryOpSlot{frozenSetContains} FrozenSetType.slots.Eq = &binaryOpSlot{frozenSetEq} FrozenSetType.slots.GE = &binaryOpSlot{frozenSetGE} FrozenSetType.slots.GT = &binaryOpSlot{frozenSetGT} // TODO: Implement hash for frozenset. FrozenSetType.slots.Hash = &unaryOpSlot{hashNotImplemented} FrozenSetType.slots.Iter = &unaryOpSlot{frozenSetIter} FrozenSetType.slots.LE = &binaryOpSlot{frozenSetLE} FrozenSetType.slots.Len = &unaryOpSlot{frozenSetLen} FrozenSetType.slots.LT = &binaryOpSlot{frozenSetLT} FrozenSetType.slots.NE = &binaryOpSlot{frozenSetNE} FrozenSetType.slots.New = &newSlot{frozenSetNew} FrozenSetType.slots.Repr = &unaryOpSlot{frozenSetRepr} } func setCompare(f *Frame, op compareOp, v *setBase, w *Object) (*Object, *BaseException) { var s2 *setBase switch { case w.isInstance(SetType): s2 = (*setBase)(toSetUnsafe(w)) case w.isInstance(FrozenSetType): s2 = (*setBase)(toFrozenSetUnsafe(w)) default: return NotImplemented, nil } if op == compareOpGE || op == compareOpGT { op = op.swapped() v, s2 = s2, v } v.dict.mutex.Lock(f) iter := newDictEntryIterator(v.dict) g1 := newDictVersionGuard(v.dict) len1 := v.dict.Len() v.dict.mutex.Unlock(f) s2.dict.mutex.Lock(f) g2 := newDictVersionGuard(s2.dict) len2 := s2.dict.Len() s2.dict.mutex.Unlock(f) result := (op != compareOpNE) switch op { case compareOpLT: if len1 >= len2 { return False.ToObject(), nil } case compareOpLE: if len1 > len2 { return False.ToObject(), nil } case compareOpEq, compareOpNE: if len1 != len2 { return GetBool(!result).ToObject(), nil } } for entry := iter.next(); entry != nil; entry = iter.next() { contains, raised := s2.contains(f, entry.key) if raised != nil { return nil, raised } if !contains { result = !result break } } if !g1.check() || !g2.check() { return nil, f.RaiseType(RuntimeErrorType, "set changed during iteration") } return GetBool(result).ToObject(), nil } func setFromSeq(f *Frame, seq *Object) (*setBase, *BaseException) { switch { case seq.isInstance(SetType): return (*setBase)(toSetUnsafe(seq)), nil case seq.isInstance(FrozenSetType): return (*setBase)(toFrozenSetUnsafe(seq)), nil } o, raised := SetType.Call(f, Args{seq}, nil) if raised != nil { return nil, raised } return (*setBase)(toSetUnsafe(o)), nil } ================================================ FILE: runtime/set_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "testing" ) func TestSetAdd(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, s *Set, args ...*Object) (*Object, *BaseException) { add, raised := GetAttr(f, s.ToObject(), NewStr("add"), nil) if raised != nil { return nil, raised } if _, raised := add.Call(f, args, nil); raised != nil { return nil, raised } return s.ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(NewSet(), "foo"), want: newTestSet("foo").ToObject()}, {args: wrapArgs(newTestSet(1, 2, 3), 2), want: newTestSet(1, 2, 3).ToObject()}, {args: wrapArgs(NewSet(), NewList()), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, {args: wrapArgs(NewSet(), "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "'add' of 'set' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSetContains(t *testing.T) { f := NewRootFrame() for _, typ := range []*Type{SetType, FrozenSetType} { cases := []invokeTestCase{ {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil)), "foo"), want: False.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestTuple(1, 2)), nil)), 2), want: True.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestTuple(3, "foo")), nil)), 42), want: False.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil)), NewList()), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(typ, "__contains__", &cas); err != "" { t.Error(err) } } } } func TestSetDiscard(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, s *Set, args ...*Object) (*Object, *BaseException) { discard, raised := GetAttr(f, s.ToObject(), NewStr("discard"), nil) if raised != nil { return nil, raised } if _, raised := discard.Call(f, args, nil); raised != nil { return nil, raised } return s.ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(newTestSet(1, 2, 3), 2), want: newTestSet(1, 3).ToObject()}, {args: wrapArgs(newTestSet("foo", 3), "foo"), want: newTestSet(3).ToObject()}, {args: wrapArgs(NewSet(), NewList()), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, {args: wrapArgs(NewSet(), "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "'discard' of 'set' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSetCompare(t *testing.T) { modifiedSet := newTestSet(0) modifiedType := newTestClass("Foo", []*Type{IntType}, newStringDict(map[string]*Object{ "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { modifiedSet.Add(f, NewStr("baz").ToObject()) return False.ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs(NewSet(), newTestSet("foo")), want: compareAllResultLT}, {args: wrapArgs(newTestSet(1, 2, 3), newTestSet(3, 2, 1)), want: compareAllResultEq}, {args: wrapArgs(newTestSet("foo", 3.14), newObject(ObjectType)), want: newTestTuple(false, false, false, true, true, true).ToObject()}, {args: wrapArgs(123, newTestSet("baz")), want: newTestTuple(true, true, false, true, false, false).ToObject()}, {args: wrapArgs(mustNotRaise(SetType.Call(NewRootFrame(), wrapArgs(newTestRange(100)), nil)), mustNotRaise(SetType.Call(NewRootFrame(), wrapArgs(newTestRange(100)), nil))), want: compareAllResultEq}, {args: wrapArgs(modifiedSet, newTestSet(newObject(modifiedType))), wantExc: mustCreateException(RuntimeErrorType, "set changed during iteration")}, {args: wrapArgs(newTestFrozenSet(), newTestFrozenSet("foo")), want: compareAllResultLT}, {args: wrapArgs(newTestFrozenSet(1, 2, 3), newTestFrozenSet(3, 2, 1)), want: compareAllResultEq}, {args: wrapArgs(newTestFrozenSet("foo", 3.14), newObject(ObjectType)), want: newTestTuple(true, true, false, true, false, false).ToObject()}, {args: wrapArgs(123, newTestFrozenSet("baz")), want: newTestTuple(false, false, false, true, true, true).ToObject()}, {args: wrapArgs(mustNotRaise(FrozenSetType.Call(NewRootFrame(), wrapArgs(newTestRange(100)), nil)), mustNotRaise(FrozenSetType.Call(NewRootFrame(), wrapArgs(newTestRange(100)), nil))), want: compareAllResultEq}, {args: wrapArgs(newTestFrozenSet(), NewSet()), want: newTestTuple(false, true, true, false, true, false).ToObject()}, {args: wrapArgs(newTestSet("foo", "bar"), newTestFrozenSet("foo", "bar")), want: newTestTuple(false, true, true, false, true, false).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestSetIsSubset(t *testing.T) { f := NewRootFrame() for _, typ := range []*Type{SetType, FrozenSetType} { cases := []invokeTestCase{ {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil)), newTestSet("foo")), want: True.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil)), newTestFrozenSet("foo")), want: True.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestTuple(1, 2)), nil)), newTestSet(2, 3)), want: False.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestTuple(1, 2)), nil)), newTestFrozenSet(2, 3)), want: False.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestTuple("foo")), nil)), newTestTuple("bar")), want: False.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestRange(42)), nil)), newTestRange(42)), want: True.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil)), 123), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil)), "foo", "bar"), wantExc: mustCreateException(TypeErrorType, fmt.Sprintf("'issubset' of '%s' requires 2 arguments", typ.Name()))}, } for _, cas := range cases { if err := runInvokeMethodTestCase(typ, "issubset", &cas); err != "" { t.Error(err) } } } } func TestSetIsSuperset(t *testing.T) { f := NewRootFrame() for _, typ := range []*Type{SetType, FrozenSetType} { cases := []invokeTestCase{ {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil)), newTestSet("foo")), want: False.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestTuple(1, 2)), nil)), newTestSet(2, 3)), want: False.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestTuple("foo")), nil)), newTestTuple("bar")), want: False.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestRange(42)), nil)), newTestRange(42)), want: True.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil)), 123), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil)), "foo", "bar"), wantExc: mustCreateException(TypeErrorType, fmt.Sprintf("'issuperset' of '%s' requires 2 arguments", typ.Name()))}, } for _, cas := range cases { if err := runInvokeMethodTestCase(typ, "issuperset", &cas); err != "" { t.Error(err) } } } } func TestSetIsTrue(t *testing.T) { f := NewRootFrame() for _, typ := range []*Type{SetType, FrozenSetType} { cases := []invokeTestCase{ {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil))), want: False.ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestTuple("foo", None)), nil))), want: True.ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(IsTrue), &cas); err != "" { t.Error(err) } } } } func TestSetIter(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, s *Object) (*Tuple, *BaseException) { var result *Tuple raised := seqApply(f, s, func(elems []*Object, _ bool) *BaseException { result = NewTuple(elems...) return nil }) if raised != nil { return nil, raised } return result, nil }) cases := []invokeTestCase{ {args: wrapArgs(NewSet()), want: NewTuple().ToObject()}, {args: wrapArgs(newTestSet(1, 2, 3)), want: newTestTuple(1, 2, 3).ToObject()}, {args: wrapArgs(newTestSet("foo", 3.14)), want: newTestTuple("foo", 3.14).ToObject()}, {args: wrapArgs(newTestFrozenSet()), want: NewTuple().ToObject()}, {args: wrapArgs(newTestFrozenSet(1, 2, 3)), want: newTestTuple(1, 2, 3).ToObject()}, {args: wrapArgs(newTestFrozenSet("foo", 3.14)), want: newTestTuple("foo", 3.14).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSetLen(t *testing.T) { f := NewRootFrame() for _, typ := range []*Type{SetType, FrozenSetType} { cases := []invokeTestCase{ {args: wrapArgs(mustNotRaise(typ.Call(f, nil, nil))), want: NewInt(0).ToObject()}, {args: wrapArgs(mustNotRaise(typ.Call(f, wrapArgs(newTestTuple(1, 2, 3)), nil))), want: NewInt(3).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(typ, "__len__", &cas); err != "" { t.Error(err) } } } } func TestSetNewInit(t *testing.T) { f := NewRootFrame() for _, typ := range []*Type{SetType, FrozenSetType} { cases := []invokeTestCase{ {want: NewSet().ToObject()}, {args: wrapArgs(newTestTuple("foo", "bar")), want: mustNotRaise(typ.Call(f, wrapArgs(newTestTuple("foo", "bar")), nil))}, {args: wrapArgs("abba"), want: mustNotRaise(typ.Call(f, wrapArgs(newTestTuple("a", "b")), nil))}, {args: wrapArgs(3.14), wantExc: mustCreateException(TypeErrorType, "'float' object is not iterable")}, {args: wrapArgs(NewTuple(), 1, 2, 3), wantExc: mustCreateException(TypeErrorType, fmt.Sprintf("%s expected at most 1 arguments, got 4", typ.Name()))}, } for _, cas := range cases { if err := runInvokeTestCase(typ.ToObject(), &cas); err != "" { t.Error(err) } } } } func TestSetRemove(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, s *Set, args ...*Object) (*Object, *BaseException) { remove, raised := GetAttr(f, s.ToObject(), NewStr("remove"), nil) if raised != nil { return nil, raised } if _, raised := remove.Call(f, args, nil); raised != nil { return nil, raised } return s.ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(newTestSet(1, 2, 3), 2), want: newTestSet(1, 3).ToObject()}, {args: wrapArgs(newTestSet("foo", 3), "foo"), want: newTestSet(3).ToObject()}, {args: wrapArgs(NewSet(), "foo"), wantExc: mustCreateException(KeyErrorType, "foo")}, {args: wrapArgs(NewSet(), NewList()), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, {args: wrapArgs(NewSet(), "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "'remove' of 'set' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSetStrRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewSet()), want: NewStr("set([])").ToObject()}, {args: wrapArgs(newTestSet("foo")), want: NewStr("set(['foo'])").ToObject()}, {args: wrapArgs(newTestSet(TupleType, ExceptionType)), want: NewStr("set([, ])").ToObject()}, {args: wrapArgs(newTestFrozenSet()), want: NewStr("frozenset([])").ToObject()}, {args: wrapArgs(newTestFrozenSet("foo")), want: NewStr("frozenset(['foo'])").ToObject()}, {args: wrapArgs(newTestFrozenSet(TupleType, ExceptionType)), want: NewStr("frozenset([, ])").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func TestSetUpdate(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, s *Set, args ...*Object) (*Object, *BaseException) { update, raised := GetAttr(f, s.ToObject(), NewStr("update"), nil) if raised != nil { return nil, raised } if _, raised := update.Call(f, args, nil); raised != nil { return nil, raised } return s.ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(NewSet(), "foo"), want: newTestSet("f", "o").ToObject()}, {args: wrapArgs(NewSet(), newTestDict(1, "1", 2, "2")), want: newTestSet(1, 2).ToObject()}, {args: wrapArgs(NewSet(), newTestTuple("foo", "bar", "bar")), want: newTestSet("foo", "bar").ToObject()}, {args: wrapArgs(NewSet(), newTestTuple(NewDict())), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'dict'")}, {args: wrapArgs(NewSet(), 123), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, {args: wrapArgs(NewSet(), "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "'update' of 'set' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func newTestSet(elems ...interface{}) *Set { f := NewRootFrame() wrappedElems, raised := seqWrapEach(f, elems...) if raised != nil { panic(raised) } s := NewSet() for _, elem := range wrappedElems { if _, raised := s.Add(f, elem); raised != nil { panic(raised) } } return s } func newTestFrozenSet(elems ...interface{}) *FrozenSet { f := NewRootFrame() wrappedElems, raised := seqWrapEach(f, elems...) if raised != nil { panic(raised) } d := NewDict() for _, elem := range wrappedElems { if raised := d.SetItem(f, elem, None); raised != nil { panic(raised) } } return &FrozenSet{Object{typ: FrozenSetType}, d} } ================================================ FILE: runtime/slice.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import "reflect" const errBadSliceIndex = "slice indices must be integers or None or have an __index__ method" var ( // SliceType is the object representing the Python 'slice' type. SliceType = newBasisType("slice", reflect.TypeOf(Slice{}), toSliceUnsafe, ObjectType) ) // Slice represents Python 'slice' objects. type Slice struct { Object start *Object `attr:"start"` stop *Object `attr:"stop"` step *Object `attr:"step"` } func toSliceUnsafe(o *Object) *Slice { return (*Slice)(o.toPointer()) } // calcSlice returns the three range indices (start, stop, step) and the length // of the slice produced by slicing a sequence of length numElems by s. As with // seqRange, the resulting indices can be used to iterate over the slice like: // // for i := start; i != stop; i += step { ... } func (s *Slice) calcSlice(f *Frame, numElems int) (int, int, int, int, *BaseException) { step := 1 if s.step != nil && s.step != None { if s.step.typ.slots.Index == nil { return 0, 0, 0, 0, f.RaiseType(TypeErrorType, errBadSliceIndex) } i, raised := IndexInt(f, s.step) if raised != nil { return 0, 0, 0, 0, raised } step = i } var startDef, stopDef int if step > 0 { startDef, stopDef = 0, numElems } else { startDef, stopDef = numElems-1, -1 } start, raised := sliceClampIndex(f, s.start, startDef, numElems) if raised != nil { return 0, 0, 0, 0, raised } stop, raised := sliceClampIndex(f, s.stop, stopDef, numElems) if raised != nil { return 0, 0, 0, 0, raised } stop, sliceLen, result := seqRange(start, stop, step) switch result { case seqRangeZeroStep: return 0, 0, 0, 0, f.RaiseType(ValueErrorType, "slice step cannot be zero") case seqRangeOverflow: return 0, 0, 0, 0, f.RaiseType(OverflowErrorType, errResultTooLarge) } return start, stop, step, sliceLen, nil } // ToObject upcasts s to an Object. func (s *Slice) ToObject() *Object { return &s.Object } func sliceEq(f *Frame, v, w *Object) (*Object, *BaseException) { return sliceCompare(f, toSliceUnsafe(v), w, Eq) } func sliceGE(f *Frame, v, w *Object) (*Object, *BaseException) { return sliceCompare(f, toSliceUnsafe(v), w, GE) } func sliceGT(f *Frame, v, w *Object) (*Object, *BaseException) { return sliceCompare(f, toSliceUnsafe(v), w, GT) } func sliceLE(f *Frame, v, w *Object) (*Object, *BaseException) { return sliceCompare(f, toSliceUnsafe(v), w, LE) } func sliceLT(f *Frame, v, w *Object) (*Object, *BaseException) { return sliceCompare(f, toSliceUnsafe(v), w, LT) } func sliceNE(f *Frame, v, w *Object) (*Object, *BaseException) { return sliceCompare(f, toSliceUnsafe(v), w, NE) } func sliceNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{ObjectType, ObjectType, ObjectType} argc := len(args) if argc >= 1 && argc <= 3 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "__new__", args, expectedTypes...); raised != nil { return nil, raised } s := toSliceUnsafe(newObject(t)) if argc == 1 { s.stop = args[0] } else { s.start = args[0] s.stop = args[1] if argc > 2 { s.step = args[2] } } return s.ToObject(), nil } func sliceRepr(f *Frame, o *Object) (*Object, *BaseException) { s := toSliceUnsafe(o) elem0, elem1, elem2 := None, s.stop, None if s.start != nil { elem0 = s.start } if s.step != nil { elem2 = s.step } r, raised := Repr(f, NewTuple3(elem0, elem1, elem2).ToObject()) if raised != nil { return nil, raised } return NewStr("slice" + r.Value()).ToObject(), nil } func initSliceType(map[string]*Object) { SliceType.flags &^= typeFlagBasetype SliceType.slots.Eq = &binaryOpSlot{sliceEq} SliceType.slots.GE = &binaryOpSlot{sliceGE} SliceType.slots.GT = &binaryOpSlot{sliceGT} SliceType.slots.Hash = &unaryOpSlot{hashNotImplemented} SliceType.slots.LE = &binaryOpSlot{sliceLE} SliceType.slots.LT = &binaryOpSlot{sliceLT} SliceType.slots.NE = &binaryOpSlot{sliceNE} SliceType.slots.New = &newSlot{sliceNew} SliceType.slots.Repr = &unaryOpSlot{sliceRepr} } func sliceClampIndex(f *Frame, index *Object, def, seqLen int) (int, *BaseException) { if index == nil || index == None { return def, nil } if index.typ.slots.Index == nil { return 0, f.RaiseType(TypeErrorType, errBadSliceIndex) } i, raised := IndexInt(f, index) if raised != nil { return 0, raised } return seqClampIndex(i, seqLen), nil } func sliceCompare(f *Frame, v *Slice, w *Object, cmp binaryOpFunc) (*Object, *BaseException) { if !w.isInstance(SliceType) { return NotImplemented, nil } rhs := toSliceUnsafe(w) elems1, elems2 := []*Object{None, v.stop, None}, []*Object{None, rhs.stop, None} if v.start != nil { elems1[0] = v.start } if v.step != nil { elems1[2] = v.step } if rhs.start != nil { elems2[0] = rhs.start } if rhs.step != nil { elems2[2] = rhs.step } return seqCompare(f, elems1, elems2, cmp) } ================================================ FILE: runtime/slice_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestSliceCalcSize(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, s *Slice, numElems int) (*Object, *BaseException) { start, stop, step, sliceLen, raised := s.calcSlice(f, numElems) if raised != nil { return nil, raised } return newTestTuple(start, stop, step, sliceLen).ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(newTestSlice(4), 6), want: newTestTuple(0, 4, 1, 4).ToObject()}, {args: wrapArgs(newTestSlice(-8), 3), want: newTestTuple(0, 0, 1, 0).ToObject()}, {args: wrapArgs(newTestSlice(0, 10), 3), want: newTestTuple(0, 3, 1, 3).ToObject()}, {args: wrapArgs(newTestSlice(1, 2, newObject(ObjectType)), 0), wantExc: mustCreateException(TypeErrorType, errBadSliceIndex)}, {args: wrapArgs(newTestSlice(newObject(ObjectType)), 10), wantExc: mustCreateException(TypeErrorType, errBadSliceIndex)}, {args: wrapArgs(newTestSlice(newObject(ObjectType), 4), 10), wantExc: mustCreateException(TypeErrorType, errBadSliceIndex)}, {args: wrapArgs(newTestSlice(1.0, 4), 10), wantExc: mustCreateException(TypeErrorType, errBadSliceIndex)}, {args: wrapArgs(newTestSlice(1, 2, 0), 3), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSliceCompare(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestSlice(None), newTestSlice(None)), want: compareAllResultEq}, {args: wrapArgs(newTestSlice(2), newTestSlice(1)), want: compareAllResultGT}, {args: wrapArgs(newTestSlice(1, 2), newTestSlice(1, 3)), want: compareAllResultLT}, {args: wrapArgs(newTestSlice(1, 2, 3), newTestSlice(1, 2)), want: compareAllResultGT}, {args: wrapArgs(None, newTestSlice(1, 2)), want: compareAllResultLT}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestSliceNew(t *testing.T) { cases := []invokeTestCase{ {args: nil, wantExc: mustCreateException(TypeErrorType, "'__new__' of 'object' requires 3 arguments")}, {args: wrapArgs(10), want: (&Slice{Object{typ: SliceType}, nil, NewInt(10).ToObject(), nil}).ToObject()}, {args: wrapArgs(1.2, "foo"), want: (&Slice{Object{typ: SliceType}, NewFloat(1.2).ToObject(), NewStr("foo").ToObject(), nil}).ToObject()}, {args: wrapArgs(None, None, true), want: (&Slice{Object{typ: SliceType}, None, None, True.ToObject()}).ToObject()}, {args: wrapArgs(1, 2, 3, 4), wantExc: mustCreateException(TypeErrorType, "'__new__' of 'object' requires 3 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(SliceType.ToObject(), &cas); err != "" { t.Error(err) } } } func TestSliceStrRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestSlice(3.14)), want: NewStr("slice(None, 3.14, None)").ToObject()}, {args: wrapArgs(newTestSlice("foo", "bar")), want: NewStr("slice('foo', 'bar', None)").ToObject()}, {args: wrapArgs(newTestSlice(1, 2, 3)), want: NewStr("slice(1, 2, 3)").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } } func newTestSlice(args ...interface{}) *Object { return mustNotRaise(SliceType.Call(NewRootFrame(), wrapArgs(args...), nil)) } ================================================ FILE: runtime/slots.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" "strings" ) var ( slotsType = reflect.TypeOf(typeSlots{}) numSlots = slotsType.NumField() slotNames = calcSlotNames() ) type slot interface { // makeCallable returns a new callable object that forwards calls to // the receiving slot with the given slotName. It is used to populate // t's type dictionary so that slots are accessible from Python. makeCallable(t *Type, slotName string) *Object // wrapCallable updates the receiver slot to forward its calls to the // given callable. This method is called when a user defined type // defines a slot method in Python to override the slot. wrapCallable(callable *Object) bool } type basisSlot struct { Fn func(*Object) reflect.Value } func (s *basisSlot) makeCallable(t *Type, slotName string) *Object { return nil } func (s *basisSlot) wrapCallable(callable *Object) bool { return false } type binaryOpFunc func(*Frame, *Object, *Object) (*Object, *BaseException) type binaryOpSlot struct { Fn binaryOpFunc } func (s *binaryOpSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t, ObjectType); raised != nil { return nil, raised } return s.Fn(f, args[0], args[1]) }).ToObject() } func (s *binaryOpSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, v, w *Object) (*Object, *BaseException) { return callable.Call(f, Args{v, w}, nil) } return true } type callSlot struct { Fn func(*Frame, *Object, Args, KWArgs) (*Object, *BaseException) } func (s *callSlot) makeCallable(t *Type, _ string) *Object { return newBuiltinFunction("__call__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodVarArgs(f, "__call__", args, t); raised != nil { return nil, raised } return t.slots.Call.Fn(f, args[0], args[1:], kwargs) }).ToObject() } func (s *callSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, o *Object, args Args, kwargs KWArgs) (*Object, *BaseException) { callArgs := make(Args, len(args)+1) callArgs[0] = o copy(callArgs[1:], args) return callable.Call(f, callArgs, kwargs) } return true } type delAttrSlot struct { Fn func(*Frame, *Object, *Str) *BaseException } func (s *delAttrSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t, StrType); raised != nil { return nil, raised } if raised := s.Fn(f, args[0], toStrUnsafe(args[1])); raised != nil { return nil, raised } return None, nil }).ToObject() } func (s *delAttrSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, o *Object, name *Str) *BaseException { _, raised := callable.Call(f, Args{o, name.ToObject()}, nil) return raised } return true } type deleteSlot struct { Fn func(*Frame, *Object, *Object) *BaseException } func (s *deleteSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t, ObjectType); raised != nil { return nil, raised } if raised := s.Fn(f, args[0], args[1]); raised != nil { return nil, raised } return None, nil }).ToObject() } func (s *deleteSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, desc *Object, inst *Object) *BaseException { _, raised := callable.Call(f, Args{desc, inst}, nil) return raised } return true } type delItemSlot struct { Fn func(*Frame, *Object, *Object) *BaseException } func (s *delItemSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t, ObjectType); raised != nil { return nil, raised } if raised := s.Fn(f, args[0], args[1]); raised != nil { return nil, raised } return None, nil }).ToObject() } func (s *delItemSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, o *Object, key *Object) *BaseException { _, raised := callable.Call(f, Args{o, key}, nil) return raised } return true } type getAttributeSlot struct { Fn func(*Frame, *Object, *Str) (*Object, *BaseException) } func (s *getAttributeSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t, StrType); raised != nil { return nil, raised } return s.Fn(f, args[0], toStrUnsafe(args[1])) }).ToObject() } func (s *getAttributeSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, o *Object, name *Str) (*Object, *BaseException) { return callable.Call(f, Args{o, name.ToObject()}, nil) } return true } type getSlot struct { Fn func(*Frame, *Object, *Object, *Type) (*Object, *BaseException) } func (s *getSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t, ObjectType, TypeType); raised != nil { return nil, raised } return s.Fn(f, args[0], args[1], toTypeUnsafe(args[2])) }).ToObject() } func (s *getSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, desc, inst *Object, owner *Type) (*Object, *BaseException) { return callable.Call(f, Args{desc, inst, owner.ToObject()}, nil) } return true } type initSlot struct { Fn func(*Frame, *Object, Args, KWArgs) (*Object, *BaseException) } func (s *initSlot) makeCallable(t *Type, _ string) *Object { return newBuiltinFunction("__init__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodVarArgs(f, "__init__", args, t); raised != nil { return nil, raised } return t.slots.Init.Fn(f, args[0], args[1:], kwargs) }).ToObject() } func (s *initSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, o *Object, args Args, kwargs KWArgs) (*Object, *BaseException) { callArgs := make(Args, len(args)+1) callArgs[0] = o copy(callArgs[1:], args) return callable.Call(f, callArgs, kwargs) } return true } type nativeSlot struct { Fn func(*Frame, *Object) (reflect.Value, *BaseException) } func (s *nativeSlot) makeCallable(t *Type, slotName string) *Object { return nil } func (s *nativeSlot) wrapCallable(callable *Object) bool { return false } type newSlot struct { Fn func(*Frame, *Type, Args, KWArgs) (*Object, *BaseException) } func (s *newSlot) makeCallable(t *Type, _ string) *Object { return newStaticMethod(newBuiltinFunction("__new__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkFunctionVarArgs(f, "__new__", args, TypeType); raised != nil { return nil, raised } typeArg := toTypeUnsafe(args[0]) if !typeArg.isSubclass(t) { format := "%[1]s.__new__(%[2]s): %[2]s is not a subtype of %[1]s" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, t.Name(), typeArg.Name())) } return t.slots.New.Fn(f, typeArg, args[1:], kwargs) }).ToObject()).ToObject() } func (s *newSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, t *Type, args Args, kwargs KWArgs) (*Object, *BaseException) { callArgs := make(Args, len(args)+1) callArgs[0] = t.ToObject() copy(callArgs[1:], args) return callable.Call(f, callArgs, kwargs) } return true } type setAttrSlot struct { Fn func(*Frame, *Object, *Str, *Object) *BaseException } func (s *setAttrSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t, StrType, ObjectType); raised != nil { return nil, raised } if raised := s.Fn(f, args[0], toStrUnsafe(args[1]), args[2]); raised != nil { return nil, raised } return None, nil }).ToObject() } func (s *setAttrSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, o *Object, name *Str, value *Object) *BaseException { _, raised := callable.Call(f, Args{o, name.ToObject(), value}, nil) return raised } return true } type setItemSlot struct { Fn func(*Frame, *Object, *Object, *Object) *BaseException } func (s *setItemSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t, ObjectType, ObjectType); raised != nil { return nil, raised } if raised := s.Fn(f, args[0], args[1], args[2]); raised != nil { return nil, raised } return None, nil }).ToObject() } func (s *setItemSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, o *Object, key *Object, value *Object) *BaseException { _, raised := callable.Call(f, Args{o, key, value}, nil) return raised } return true } type setSlot struct { Fn func(*Frame, *Object, *Object, *Object) *BaseException } func (s *setSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t, ObjectType, ObjectType); raised != nil { return nil, raised } if raised := s.Fn(f, args[0], args[1], args[2]); raised != nil { return nil, raised } return None, nil }).ToObject() } func (s *setSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, desc, inst, value *Object) *BaseException { _, raised := callable.Call(f, Args{desc, inst, value}, nil) return raised } return true } type unaryOpSlot struct { Fn func(*Frame, *Object) (*Object, *BaseException) } func (s *unaryOpSlot) makeCallable(t *Type, slotName string) *Object { return newBuiltinFunction(slotName, func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, slotName, args, t); raised != nil { return nil, raised } return s.Fn(f, args[0]) }).ToObject() } func (s *unaryOpSlot) wrapCallable(callable *Object) bool { s.Fn = func(f *Frame, o *Object) (*Object, *BaseException) { return callable.Call(f, Args{o}, nil) } return true } // typeSlots hold a type's special methods such as __eq__. During type // initialization, any field that is not set for that type will be inherited // according to the type's MRO. Therefore, any given field will be nil only if // that method is not defined for the type nor any of its super classes. Each // slot is expected to be a pointer to a struct with a single function field. // The wrapper structs permit comparison of like slots which is occasionally // necessary to determine whether a function has been overridden by a subclass. type typeSlots struct { Abs *unaryOpSlot Add *binaryOpSlot And *binaryOpSlot Basis *basisSlot Call *callSlot Cmp *binaryOpSlot Complex *unaryOpSlot Contains *binaryOpSlot DelAttr *delAttrSlot Delete *deleteSlot DelItem *delItemSlot Div *binaryOpSlot DivMod *binaryOpSlot Eq *binaryOpSlot Float *unaryOpSlot FloorDiv *binaryOpSlot GE *binaryOpSlot Get *getSlot GetAttribute *getAttributeSlot GetItem *binaryOpSlot GT *binaryOpSlot Hash *unaryOpSlot Hex *unaryOpSlot IAdd *binaryOpSlot IAnd *binaryOpSlot IDiv *binaryOpSlot IDivMod *binaryOpSlot IFloorDiv *binaryOpSlot ILShift *binaryOpSlot IMod *binaryOpSlot IMul *binaryOpSlot Index *unaryOpSlot Init *initSlot Int *unaryOpSlot Invert *unaryOpSlot IOr *binaryOpSlot IPow *binaryOpSlot IRShift *binaryOpSlot ISub *binaryOpSlot Iter *unaryOpSlot IXor *binaryOpSlot LE *binaryOpSlot Len *unaryOpSlot Long *unaryOpSlot LShift *binaryOpSlot LT *binaryOpSlot Mod *binaryOpSlot Mul *binaryOpSlot Native *nativeSlot NE *binaryOpSlot Neg *unaryOpSlot New *newSlot Next *unaryOpSlot NonZero *unaryOpSlot Oct *unaryOpSlot Or *binaryOpSlot Pos *unaryOpSlot Pow *binaryOpSlot RAdd *binaryOpSlot RAnd *binaryOpSlot RDiv *binaryOpSlot RDivMod *binaryOpSlot Repr *unaryOpSlot RFloorDiv *binaryOpSlot RLShift *binaryOpSlot RMod *binaryOpSlot RMul *binaryOpSlot ROr *binaryOpSlot RPow *binaryOpSlot RRShift *binaryOpSlot RShift *binaryOpSlot RSub *binaryOpSlot RXor *binaryOpSlot Set *setSlot SetAttr *setAttrSlot SetItem *setItemSlot Str *unaryOpSlot Sub *binaryOpSlot Unicode *unaryOpSlot Xor *binaryOpSlot } func calcSlotNames() []string { names := make([]string, numSlots, numSlots) for i := 0; i < numSlots; i++ { field := slotsType.Field(i) name := "" if field.Name == "Next" { name = "next" } else { name = fmt.Sprintf("__%s__", strings.ToLower(field.Name)) } names[i] = name } return names } ================================================ FILE: runtime/slots_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "reflect" "testing" ) func TestSlotMakeCallable(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, NewDict()) foo := newObject(fooType) // fun returns a tuple: (ret, args) where ret is the return value of // the callable produced by makeCallable and args are the arguments // that were passed into the callable. fun := wrapFuncForTest(func(f *Frame, s slot, ret *Object, args ...*Object) (*Object, *BaseException) { gotArgs := None prepareTestSlot(s, &gotArgs, ret) callable := s.makeCallable(fooType, "__slot__") if callable == nil { // This slot does not produce a callable, so just // return None. return None, nil } result, raised := callable.Call(f, args, nil) if raised != nil { return nil, raised } return NewTuple(result, gotArgs).ToObject(), nil }) cases := []invokeTestCase{ {args: wrapArgs(&basisSlot{}, None), want: None}, {args: wrapArgs(&binaryOpSlot{}, "foo", foo, 123), want: newTestTuple("foo", newTestTuple(foo, 123)).ToObject()}, {args: wrapArgs(&binaryOpSlot{}, None, "abc", 123), wantExc: mustCreateException(TypeErrorType, "'__slot__' requires a 'Foo' object but received a 'str'")}, {args: wrapArgs(&delAttrSlot{}, None, foo, "bar"), want: newTestTuple(None, newTestTuple(foo, "bar")).ToObject()}, {args: wrapArgs(&delAttrSlot{}, None, foo, 3.14), wantExc: mustCreateException(TypeErrorType, "'__slot__' requires a 'str' object but received a 'float'")}, {args: wrapArgs(&delAttrSlot{}, RuntimeErrorType, foo, "bar"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&deleteSlot{}, None, foo, "bar"), want: newTestTuple(None, newTestTuple(foo, "bar")).ToObject()}, {args: wrapArgs(&deleteSlot{}, None, foo, 1, 2, 3), wantExc: mustCreateException(TypeErrorType, "'__slot__' of 'Foo' requires 2 arguments")}, {args: wrapArgs(&deleteSlot{}, RuntimeErrorType, foo, "bar"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&delItemSlot{}, None, foo, "bar"), want: newTestTuple(None, newTestTuple(foo, "bar")).ToObject()}, {args: wrapArgs(&delItemSlot{}, None, foo, 1, 2, 3), wantExc: mustCreateException(TypeErrorType, "'__slot__' of 'Foo' requires 2 arguments")}, {args: wrapArgs(&delItemSlot{}, RuntimeErrorType, foo, "bar"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&getAttributeSlot{}, None, foo, "bar"), want: newTestTuple(None, newTestTuple(foo, "bar")).ToObject()}, {args: wrapArgs(&getAttributeSlot{}, None, foo, 3.14), wantExc: mustCreateException(TypeErrorType, "'__slot__' requires a 'str' object but received a 'float'")}, {args: wrapArgs(&getAttributeSlot{}, RuntimeErrorType, foo, "bar"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&getSlot{}, 3.14, foo, 123, IntType), want: newTestTuple(3.14, newTestTuple(foo, 123, IntType)).ToObject()}, {args: wrapArgs(&getSlot{}, None, foo, "bar", "baz"), wantExc: mustCreateException(TypeErrorType, "'__slot__' requires a 'type' object but received a 'str'")}, {args: wrapArgs(&nativeSlot{}, None), want: None}, {args: wrapArgs(&setAttrSlot{}, None, foo, "bar", 123), want: newTestTuple(None, newTestTuple(foo, "bar", 123)).ToObject()}, {args: wrapArgs(&setAttrSlot{}, None, foo, true, None), wantExc: mustCreateException(TypeErrorType, "'__slot__' requires a 'str' object but received a 'bool'")}, {args: wrapArgs(&setAttrSlot{}, RuntimeErrorType, foo, "bar", "baz"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&setItemSlot{}, None, foo, "bar", true), want: newTestTuple(None, newTestTuple(foo, "bar", true)).ToObject()}, {args: wrapArgs(&setItemSlot{}, None, foo, 1, 2, 3), wantExc: mustCreateException(TypeErrorType, "'__slot__' of 'Foo' requires 3 arguments")}, {args: wrapArgs(&setItemSlot{}, RuntimeErrorType, foo, "bar", "baz"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&setSlot{}, None, foo, 3.14, false), want: newTestTuple(None, newTestTuple(foo, 3.14, false)).ToObject()}, {args: wrapArgs(&setSlot{}, RuntimeErrorType, foo, "bar", "baz"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&setSlot{}, None, foo, 1, 2, 3), wantExc: mustCreateException(TypeErrorType, "'__slot__' of 'Foo' requires 3 arguments")}, {args: wrapArgs(&unaryOpSlot{}, 42, foo), want: newTestTuple(42, NewTuple(foo)).ToObject()}, {args: wrapArgs(&unaryOpSlot{}, None, foo, "bar"), wantExc: mustCreateException(TypeErrorType, "'__slot__' of 'Foo' requires 1 arguments")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestSlotWrapCallable(t *testing.T) { // fun returns a tuple: (ret, args, kwargs) where ret is the return // value of the slot, args and kwargs are the positional and keyword // parameters passed to it. fun := wrapFuncForTest(func(f *Frame, s slot, ret *Object, args ...*Object) (*Object, *BaseException) { gotArgs := None gotKWArgs := None wrapped := newBuiltinFunction("wrapped", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if ret.isInstance(TypeType) && toTypeUnsafe(ret).isSubclass(BaseExceptionType) { return nil, f.Raise(ret, nil, nil) } gotArgs = NewTuple(args.makeCopy()...).ToObject() gotKWArgs = kwargs.makeDict().ToObject() return ret, nil }).ToObject() s.wrapCallable(wrapped) fnField := reflect.ValueOf(s).Elem().Field(0) if fnField.IsNil() { // Return None to denote the slot was empty. return None, nil } // Wrap the underlying slot function (s.Fn) and call it. This // is more convenient than using reflection to call it. fn, raised := WrapNative(f, fnField) if raised != nil { return nil, raised } result, raised := fn.Call(f, append(Args{f.ToObject()}, args...), nil) if raised != nil { return nil, raised } return NewTuple(result, gotArgs, gotKWArgs).ToObject(), nil }) o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(&basisSlot{}, "no"), want: None}, {args: wrapArgs(&binaryOpSlot{}, "ret", "foo", "bar"), want: newTestTuple("ret", newTestTuple("foo", "bar"), NewDict()).ToObject()}, {args: wrapArgs(&binaryOpSlot{}, RuntimeErrorType, "foo", "bar"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&callSlot{}, "ret", true, wrapArgs(1, 2), None), want: newTestTuple("ret", newTestTuple(true, 1, 2), NewDict()).ToObject()}, {args: wrapArgs(&callSlot{}, "ret", "foo", None, wrapKWArgs("a", "b")), want: newTestTuple("ret", newTestTuple("foo"), newTestDict("a", "b")).ToObject()}, {args: wrapArgs(&callSlot{}, "ret", 3.14, wrapArgs(false), wrapKWArgs("foo", 42)), want: newTestTuple("ret", newTestTuple(3.14, false), newTestDict("foo", 42)).ToObject()}, {args: wrapArgs(&callSlot{}, RuntimeErrorType, true, wrapArgs(1, 2), None), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&delAttrSlot{}, "ret", o, "foo"), want: newTestTuple(None, newTestTuple(o, "foo"), NewDict()).ToObject()}, {args: wrapArgs(&delAttrSlot{}, RuntimeErrorType, o, "foo"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&deleteSlot{}, "ret", o, 3.14), want: newTestTuple(None, newTestTuple(o, 3.14), NewDict()).ToObject()}, {args: wrapArgs(&deleteSlot{}, RuntimeErrorType, o, 3.14), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&delItemSlot{}, "ret", o, false), want: newTestTuple(None, newTestTuple(o, false), NewDict()).ToObject()}, {args: wrapArgs(&delItemSlot{}, RuntimeErrorType, o, false), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&getAttributeSlot{}, "ret", o, "foo"), want: newTestTuple("ret", newTestTuple(o, "foo"), NewDict()).ToObject()}, {args: wrapArgs(&getAttributeSlot{}, RuntimeErrorType, o, "foo"), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&getSlot{}, "ret", o, "foo", SetType), want: newTestTuple("ret", newTestTuple(o, "foo", SetType), NewDict()).ToObject()}, {args: wrapArgs(&getSlot{}, RuntimeErrorType, o, "foo", SetType), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&initSlot{}, "ret", true, wrapArgs(1, 2), None), want: newTestTuple("ret", newTestTuple(true, 1, 2), NewDict()).ToObject()}, {args: wrapArgs(&initSlot{}, "ret", "foo", None, wrapKWArgs("a", "b")), want: newTestTuple("ret", newTestTuple("foo"), newTestDict("a", "b")).ToObject()}, {args: wrapArgs(&initSlot{}, "ret", 3.14, wrapArgs(false), wrapKWArgs("foo", 42)), want: newTestTuple("ret", newTestTuple(3.14, false), newTestDict("foo", 42)).ToObject()}, {args: wrapArgs(&initSlot{}, RuntimeErrorType, true, wrapArgs(1, 2), None), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&nativeSlot{}, "no"), want: None}, {args: wrapArgs(&newSlot{}, "ret", StrType, wrapArgs(1, 2), None), want: newTestTuple("ret", newTestTuple(StrType, 1, 2), NewDict()).ToObject()}, {args: wrapArgs(&newSlot{}, "ret", ObjectType, None, wrapKWArgs("a", "b")), want: newTestTuple("ret", newTestTuple(ObjectType), newTestDict("a", "b")).ToObject()}, {args: wrapArgs(&newSlot{}, "ret", ListType, wrapArgs(false), wrapKWArgs("foo", 42)), want: newTestTuple("ret", newTestTuple(ListType, false), newTestDict("foo", 42)).ToObject()}, {args: wrapArgs(&newSlot{}, RuntimeErrorType, IntType, wrapArgs(1, 2), None), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&setAttrSlot{}, "ret", o, "foo", 42), want: newTestTuple(None, newTestTuple(o, "foo", 42), NewDict()).ToObject()}, {args: wrapArgs(&setAttrSlot{}, RuntimeErrorType, o, "foo", 42), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&setItemSlot{}, "ret", o, "foo", 42), want: newTestTuple(None, newTestTuple(o, "foo", 42), NewDict()).ToObject()}, {args: wrapArgs(&setItemSlot{}, RuntimeErrorType, o, "foo", 42), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&setSlot{}, "ret", o, "foo", 42), want: newTestTuple(None, newTestTuple(o, "foo", 42), NewDict()).ToObject()}, {args: wrapArgs(&setSlot{}, RuntimeErrorType, o, "foo", 42), wantExc: mustCreateException(RuntimeErrorType, "")}, {args: wrapArgs(&unaryOpSlot{}, "ret", "foo"), want: newTestTuple("ret", newTestTuple("foo"), NewDict()).ToObject()}, {args: wrapArgs(&unaryOpSlot{}, RuntimeErrorType, "foo"), wantExc: mustCreateException(RuntimeErrorType, "")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } // prepareTestSlot sets the Fn field of s to a function that assigns its // parameters to gotArgs and returns ret. If ret is type inheriting from // BaseException, an exception of that type will be raised instead. func prepareTestSlot(s slot, gotArgs **Object, ret *Object) { fnField := reflect.ValueOf(s).Elem().Field(0) slotFuncType := fnField.Type() numIn := slotFuncType.NumIn() numOut := slotFuncType.NumOut() fnField.Set(reflect.MakeFunc(slotFuncType, func(argValues []reflect.Value) []reflect.Value { f := argValues[0].Interface().(*Frame) var raised *BaseException if ret.isInstance(TypeType) && toTypeUnsafe(ret).isSubclass(BaseExceptionType) { raised = f.Raise(ret, nil, nil) } else { // Copy the input args into *gotArgs. elems := make([]*Object, numIn-1) for i := 1; i < numIn; i++ { if elems[i-1], raised = WrapNative(f, argValues[i]); raised != nil { break } } if raised == nil { *gotArgs = NewTuple(elems...).ToObject() } } raisedValue := reflect.ValueOf(raised) if numOut == 1 { // This slot does only returns an exception so return // raised (which may be nil). return []reflect.Value{raisedValue} } // Slot returns a single value and an exception. retValue := reflect.ValueOf((*Object)(nil)) if raised == nil { retValue = reflect.ValueOf(ret) } return []reflect.Value{retValue, raisedValue} })) } ================================================ FILE: runtime/str.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "bytes" "fmt" "reflect" "regexp" "strconv" "strings" "sync/atomic" "unicode" "unicode/utf8" "unsafe" ) var ( // StrType is the object representing the Python 'str' type. StrType = newBasisType("str", reflect.TypeOf(Str{}), toStrUnsafe, BaseStringType) whitespaceSplitRegexp = regexp.MustCompile(`\s+`) strASCIISpaces = []byte(" \t\n\v\f\r") strInterpolationRegexp = regexp.MustCompile(`^%([#0 +-]?)((\*|[0-9]+)?)((\.(\*|[0-9]+))?)[hlL]?([diouxXeEfFgGcrs%])`) internedStrs = map[string]*Str{} caseOffset = byte('a' - 'A') internedName = NewStr("__name__") ) type stripSide int const ( stripSideLeft stripSide = iota stripSideRight stripSideBoth ) // InternStr adds s to the interned string map. Subsequent calls to NewStr() // will return the same underlying Str. InternStr is not thread safe and should // only be called during module initialization time. func InternStr(s string) *Str { str, _ := internedStrs[s] if str == nil { str = &Str{Object: Object{typ: StrType}, value: s, hash: NewInt(hashString(s))} internedStrs[s] = str } return str } // Str represents Python 'str' objects. type Str struct { Object value string hash *Int } // NewStr returns a new Str holding the given string value. func NewStr(value string) *Str { if s := internedStrs[value]; s != nil { return s } return &Str{Object: Object{typ: StrType}, value: value} } func toStrUnsafe(o *Object) *Str { return (*Str)(o.toPointer()) } // Decode produces a unicode object from the bytes of s assuming they have the // given encoding. Invalid code points are resolved using a strategy given by // errors: "ignore" will bypass them, "replace" will substitute the Unicode // replacement character (U+FFFD) and "strict" will raise UnicodeDecodeError. // // NOTE: Decoding UTF-8 data containing surrogates (e.g. U+D800 encoded as // '\xed\xa0\x80') will raise UnicodeDecodeError consistent with CPython 3.x // but different than 2.x. func (s *Str) Decode(f *Frame, encoding, errors string) (*Unicode, *BaseException) { // TODO: Support custom encodings and error handlers. normalized := normalizeEncoding(encoding) if normalized != "utf8" { return nil, f.RaiseType(LookupErrorType, fmt.Sprintf("unknown encoding: %s", encoding)) } var runes []rune for pos, r := range s.Value() { switch { case r != utf8.RuneError: runes = append(runes, r) case errors == EncodeIgnore: // Do nothing case errors == EncodeReplace: runes = append(runes, unicode.ReplacementChar) case errors == EncodeStrict: format := "'%s' codec can't decode byte 0x%02x in position %d" return nil, f.RaiseType(UnicodeDecodeErrorType, fmt.Sprintf(format, encoding, int(s.Value()[pos]), pos)) default: format := "unknown error handler name '%s'" return nil, f.RaiseType(LookupErrorType, fmt.Sprintf(format, errors)) } } return NewUnicodeFromRunes(runes), nil } // ToObject upcasts s to an Object. func (s *Str) ToObject() *Object { return &s.Object } // Value returns the underlying string value held by s. func (s *Str) Value() string { return s.value } func hashString(s string) int { l := len(s) if l == 0 { return 0 } h := int(s[0]) << 7 for i := 0; i < l; i++ { h = (1000003 * h) ^ int(s[i]) } h ^= l if h == -1 { h = -2 } return h } func strAdd(f *Frame, v, w *Object) (*Object, *BaseException) { if w.isInstance(UnicodeType) { // CPython explicitly dispatches to unicode here so that's how // we do it even though it would seem more natural to override // unicode.__radd__. ret, raised := toStrUnsafe(v).Decode(f, EncodeDefault, EncodeStrict) if raised != nil { return nil, raised } return unicodeAdd(f, ret.ToObject(), w) } if !w.isInstance(StrType) { return NotImplemented, nil } stringV, stringW := toStrUnsafe(v).Value(), toStrUnsafe(w).Value() if len(stringV)+len(stringW) < 0 { // This indicates an int overflow. return nil, f.RaiseType(OverflowErrorType, errResultTooLarge) } return NewStr(stringV + stringW).ToObject(), nil } func strCapitalize(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "capitalize", args, StrType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() numBytes := len(s) if numBytes == 0 { return args[0], nil } b := make([]byte, numBytes) b[0] = toUpper(s[0]) for i := 1; i < numBytes; i++ { b[i] = toLower(s[i]) } return NewStr(string(b)).ToObject(), nil } func strCenter(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { s, width, fill, raised := strJustDecodeArgs(f, args, "center") if raised != nil { return nil, raised } if len(s) >= width { return NewStr(s).ToObject(), nil } marg := width - len(s) left := marg/2 + (marg & width & 1) return NewStr(pad(s, left, marg-left, fill)).ToObject(), nil } func strContains(f *Frame, o *Object, value *Object) (*Object, *BaseException) { if value.isInstance(UnicodeType) { decoded, raised := toStrUnsafe(o).Decode(f, EncodeDefault, EncodeStrict) if raised != nil { return nil, raised } return unicodeContains(f, decoded.ToObject(), value) } if !value.isInstance(StrType) { format := "'in ' requires string as left operand, not %s" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, value.typ.Name())) } return GetBool(strings.Contains(toStrUnsafe(o).Value(), toStrUnsafe(value).Value())).ToObject(), nil } func strCount(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "count", args, StrType, ObjectType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() sep := toStrUnsafe(args[1]).Value() cnt := strings.Count(s, sep) return NewInt(cnt).ToObject(), nil } func strDecode(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { // TODO: Accept unicode for encoding and errors args. expectedTypes := []*Type{StrType, StrType, StrType} argc := len(args) if argc >= 1 && argc < 3 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "decode", args, expectedTypes...); raised != nil { return nil, raised } encoding := EncodeDefault if argc > 1 { encoding = toStrUnsafe(args[1]).Value() } errors := EncodeStrict if argc > 2 { errors = toStrUnsafe(args[2]).Value() } s, raised := toStrUnsafe(args[0]).Decode(f, encoding, errors) if raised != nil { return nil, raised } return s.ToObject(), nil } func strEndsWith(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return strStartsEndsWith(f, "endswith", args) } func strEq(f *Frame, v, w *Object) (*Object, *BaseException) { return strCompare(v, w, False, True, False), nil } // strFind returns the lowest index in s where the substring sub is found such // that sub is wholly contained in s[start:end]. Return -1 on failure. func strFind(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return strFindOrIndex(f, args, func(s, sub string) (int, *BaseException) { return strings.Index(s, sub), nil }) } func strGE(f *Frame, v, w *Object) (*Object, *BaseException) { return strCompare(v, w, False, True, True), nil } // strGetItem returns a slice of string depending on whether index is an integer // or a slice. If index is neither of those types then a TypeError is returned. func strGetItem(f *Frame, o, key *Object) (*Object, *BaseException) { s := toStrUnsafe(o).Value() switch { case key.typ.slots.Index != nil: index, raised := IndexInt(f, key) if raised != nil { return nil, raised } index, raised = seqCheckedIndex(f, len(s), index) if raised != nil { return nil, raised } return NewStr(s[index : index+1]).ToObject(), nil case key.isInstance(SliceType): slice := toSliceUnsafe(key) start, stop, step, sliceLen, raised := slice.calcSlice(f, len(s)) if raised != nil { return nil, raised } if step == 1 { return NewStr(s[start:stop]).ToObject(), nil } result := make([]byte, 0, sliceLen) for j := start; j != stop; j += step { result = append(result, s[j]) } return NewStr(string(result)).ToObject(), nil } return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("string indices must be integers or slice, not %s", key.typ.Name())) } func strGetNewArgs(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__getnewargs__", args, StrType); raised != nil { return nil, raised } return NewTuple1(args[0]).ToObject(), nil } func strGT(f *Frame, v, w *Object) (*Object, *BaseException) { return strCompare(v, w, False, False, True), nil } func strHash(f *Frame, o *Object) (*Object, *BaseException) { s := toStrUnsafe(o) p := (*unsafe.Pointer)(unsafe.Pointer(&s.hash)) if v := atomic.LoadPointer(p); v != unsafe.Pointer(nil) { return (*Int)(v).ToObject(), nil } h := NewInt(hashString(toStrUnsafe(o).Value())) atomic.StorePointer(p, unsafe.Pointer(h)) return h.ToObject(), nil } func strIndex(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return strFindOrIndex(f, args, func(s, sub string) (i int, raised *BaseException) { i = strings.Index(s, sub) if i == -1 { raised = f.RaiseType(ValueErrorType, "substring not found") } return i, raised }) } func strIsAlNum(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "isalnum", args, StrType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() if len(s) == 0 { return False.ToObject(), nil } for i := range s { if !isAlNum(s[i]) { return False.ToObject(), nil } } return True.ToObject(), nil } func strIsAlpha(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "isalpha", args, StrType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() if len(s) == 0 { return False.ToObject(), nil } for i := range s { if !isAlpha(s[i]) { return False.ToObject(), nil } } return True.ToObject(), nil } func strIsDigit(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "isdigit", args, StrType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() if len(s) == 0 { return False.ToObject(), nil } for i := range s { if !isDigit(s[i]) { return False.ToObject(), nil } } return True.ToObject(), nil } func strIsLower(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "islower", args, StrType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() if len(s) == 0 { return False.ToObject(), nil } for i := range s { if !isLower(s[i]) { return False.ToObject(), nil } } return True.ToObject(), nil } func strIsSpace(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "isspace", args, StrType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() if len(s) == 0 { return False.ToObject(), nil } for i := range s { if !isSpace(s[i]) { return False.ToObject(), nil } } return True.ToObject(), nil } func strIsTitle(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "istitle", args, StrType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() if len(s) == 0 { return False.ToObject(), nil } if len(s) == 1 { return GetBool(isUpper(s[0])).ToObject(), nil } cased := false previousIsCased := false for i := range s { if isUpper(s[i]) { if previousIsCased { return False.ToObject(), nil } previousIsCased = true cased = true } else if isLower(s[i]) { if !previousIsCased { return False.ToObject(), nil } previousIsCased = true cased = true } else { previousIsCased = false } } return GetBool(cased).ToObject(), nil } func strIsUpper(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "isupper", args, StrType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() if len(s) == 0 { return False.ToObject(), nil } for i := range s { if !isUpper(s[i]) { return False.ToObject(), nil } } return True.ToObject(), nil } func strJoin(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "join", args, StrType, ObjectType); raised != nil { return nil, raised } sep := toStrUnsafe(args[0]).Value() var result *Object raised := seqApply(f, args[1], func(parts []*Object, _ bool) *BaseException { numParts := len(parts) if numParts == 0 { result = NewStr("").ToObject() return nil } // Calculate the size of the required buffer. numChars := (numParts - 1) * len(sep) for i, part := range parts { if part.isInstance(StrType) { numChars += len(toStrUnsafe(part).Value()) } else if part.isInstance(UnicodeType) { // Some element was unicode so use the unicode // implementation. var raised *BaseException s, raised := unicodeCoerce(f, args[0]) if raised != nil { return raised } result, raised = unicodeJoinParts(f, s, parts) return raised } else { format := "sequence item %d: expected string, %s found" return f.RaiseType(TypeErrorType, fmt.Sprintf(format, i, part.typ.Name())) } } // Piece together the result string into buf. buf := bytes.Buffer{} buf.Grow(numChars) for i, part := range parts { if i > 0 { buf.WriteString(sep) } buf.WriteString(toStrUnsafe(part).Value()) } result = NewStr(buf.String()).ToObject() return nil }) if raised != nil { return nil, raised } return result, nil } func strLE(f *Frame, v, w *Object) (*Object, *BaseException) { return strCompare(v, w, True, True, False), nil } func strLen(f *Frame, o *Object) (*Object, *BaseException) { return NewInt(len(toStrUnsafe(o).Value())).ToObject(), nil } func strLJust(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { s, width, fill, raised := strJustDecodeArgs(f, args, "ljust") if raised != nil { return nil, raised } if len(s) >= width { return NewStr(s).ToObject(), nil } return NewStr(pad(s, 0, width-len(s), fill)).ToObject(), nil } func strLower(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{StrType} if raised := checkMethodArgs(f, "lower", args, expectedTypes...); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() numBytes := len(s) if numBytes == 0 { return args[0], nil } b := make([]byte, numBytes) for i := 0; i < numBytes; i++ { b[i] = toLower(s[i]) } return NewStr(string(b)).ToObject(), nil } func strLStrip(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return strStripImpl(f, args, stripSideLeft) } func strLT(f *Frame, v, w *Object) (*Object, *BaseException) { return strCompare(v, w, True, False, False), nil } func strMod(f *Frame, v, w *Object) (*Object, *BaseException) { s := toStrUnsafe(v).Value() switch { case w.isInstance(DictType): return nil, f.RaiseType(NotImplementedErrorType, "mappings not yet supported") case w.isInstance(TupleType): return strInterpolate(f, s, toTupleUnsafe(w)) default: return strInterpolate(f, s, NewTuple1(w)) } } func strMul(f *Frame, v, w *Object) (*Object, *BaseException) { s := toStrUnsafe(v).Value() n, ok, raised := strRepeatCount(f, len(s), w) if raised != nil { return nil, raised } if !ok { return NotImplemented, nil } return NewStr(strings.Repeat(s, n)).ToObject(), nil } func strNative(f *Frame, o *Object) (reflect.Value, *BaseException) { return reflect.ValueOf(toStrUnsafe(o).Value()), nil } func strNE(f *Frame, v, w *Object) (*Object, *BaseException) { return strCompare(v, w, True, False, True), nil } func strNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { if t != StrType { // Allocate a plain str and then copy it's value into an object // of the str subtype. s, raised := strNew(f, StrType, args, nil) if raised != nil { return nil, raised } result := toStrUnsafe(newObject(t)) result.value = toStrUnsafe(s).Value() return result.ToObject(), nil } argc := len(args) if argc == 0 { // Empty string. return newObject(t), nil } if argc != 1 { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("str() takes at most 1 argument (%d given)", argc)) } o := args[0] if str := o.typ.slots.Str; str != nil { result, raised := str.Fn(f, o) if raised != nil { return nil, raised } if !result.isInstance(StrType) { format := "__str__ returned non-string (type %s)" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, result.typ.Name())) } return result, nil } s, raised := Repr(f, o) if raised != nil { return nil, raised } return s.ToObject(), nil } // strReplace returns a copy of the string s with the first n non-overlapping // instances of old replaced by sub. If old is empty, it matches at the // beginning of the string. If n < 0, there is no limit on the number of // replacements. func strReplace(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { var raised *BaseException // TODO: Support unicode replace. expectedTypes := []*Type{StrType, StrType, StrType, ObjectType} argc := len(args) if argc == 3 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "replace", args, expectedTypes...); raised != nil { return nil, raised } n := -1 if argc == 4 { n, raised = ToIntValue(f, args[3]) if raised != nil { return nil, raised } } s := toStrUnsafe(args[0]).Value() // Returns early if no need to replace. if n == 0 { return NewStr(s).ToObject(), nil } old := toStrUnsafe(args[1]).Value() sub := toStrUnsafe(args[2]).Value() numBytes := len(s) // Even if s and old is blank, replace should return sub, except n is negative. // This is CPython specific behavior. if numBytes == 0 && old == "" && n >= 0 { return NewStr("").ToObject(), nil } // If old is non-blank, pass to strings.Replace. if len(old) > 0 { return NewStr(strings.Replace(s, old, sub, n)).ToObject(), nil } // If old is blank, insert sub after every bytes on s and beginning. if n < 0 { n = numBytes + 1 } // Insert sub at beginning. buf := bytes.Buffer{} buf.WriteString(sub) n-- // Insert after every byte. i := 0 for n > 0 && i < numBytes { buf.WriteByte(s[i]) buf.WriteString(sub) i++ n-- } // Write the remaining string. if i < numBytes { buf.WriteString(s[i:]) } return NewStr(buf.String()).ToObject(), nil } func strRepr(_ *Frame, o *Object) (*Object, *BaseException) { s := toStrUnsafe(o).Value() buf := bytes.Buffer{} buf.WriteRune('\'') numBytes := len(s) for i := 0; i < numBytes; i++ { r := rune(s[i]) if escape, ok := escapeMap[r]; ok { buf.WriteString(escape) } else if r > unicode.MaxASCII || !unicode.IsPrint(r) { buf.WriteString(fmt.Sprintf(`\x%02x`, r)) } else { buf.WriteRune(r) } } buf.WriteRune('\'') return NewStr(buf.String()).ToObject(), nil } func strRFind(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return strFindOrIndex(f, args, func(s, sub string) (int, *BaseException) { return strings.LastIndex(s, sub), nil }) } func strRIndex(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return strFindOrIndex(f, args, func(s, sub string) (i int, raised *BaseException) { i = strings.LastIndex(s, sub) if i == -1 { raised = f.RaiseType(ValueErrorType, "substring not found") } return i, raised }) } func strRJust(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { s, width, fill, raised := strJustDecodeArgs(f, args, "rjust") if raised != nil { return nil, raised } if len(s) >= width { return NewStr(s).ToObject(), nil } return NewStr(pad(s, width-len(s), 0, fill)).ToObject(), nil } func strSplit(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{StrType, ObjectType, IntType} argc := len(args) if argc == 1 || argc == 2 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "split", args, expectedTypes...); raised != nil { return nil, raised } sep := "" if argc > 1 { if arg1 := args[1]; arg1.isInstance(StrType) { sep = toStrUnsafe(arg1).Value() if sep == "" { return nil, f.RaiseType(ValueErrorType, "empty separator") } } else if arg1 != None { return nil, f.RaiseType(TypeErrorType, "expected a str separator") } } maxSplit := -1 if argc > 2 { if i := toIntUnsafe(args[2]).Value(); i >= 0 { maxSplit = i + 1 } } s := toStrUnsafe(args[0]).Value() var parts []string if sep == "" { s = strings.TrimLeft(s, string(strASCIISpaces)) parts = whitespaceSplitRegexp.Split(s, maxSplit) l := len(parts) if l > 0 && strings.Trim(parts[l-1], string(strASCIISpaces)) == "" { parts = parts[:l-1] } } else { parts = strings.SplitN(s, sep, maxSplit) } results := make([]*Object, len(parts)) for i, part := range parts { results[i] = NewStr(part).ToObject() } return NewList(results...).ToObject(), nil } func strSplitLines(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{StrType, ObjectType} argc := len(args) if argc == 1 { expectedTypes = expectedTypes[:1] } if raised := checkMethodArgs(f, "splitlines", args, expectedTypes...); raised != nil { return nil, raised } keepEnds := false if argc == 2 { i, raised := ToIntValue(f, args[1]) if raised != nil { return nil, raised } keepEnds = i != 0 } s := toStrUnsafe(args[0]).Value() numChars := len(s) start, end := 0, 0 lines := make([]*Object, 0, 2) for start < numChars { eol := 0 for end = start; end < numChars; end++ { c := s[end] if c == '\n' { eol = end + 1 break } if c == '\r' { eol = end + 1 if eol < numChars && s[eol] == '\n' { eol++ } break } } if end >= numChars { eol = end } line := "" if keepEnds { line = s[start:eol] } else { line = s[start:end] } lines = append(lines, NewStr(line).ToObject()) start = eol } return NewList(lines...).ToObject(), nil } func strStrip(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return strStripImpl(f, args, stripSideBoth) } func strRStrip(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return strStripImpl(f, args, stripSideRight) } func strStripImpl(f *Frame, args Args, side stripSide) (*Object, *BaseException) { expectedTypes := []*Type{StrType, ObjectType} argc := len(args) if argc == 1 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "strip", args, expectedTypes...); raised != nil { return nil, raised } s := toStrUnsafe(args[0]) charsArg := None if argc > 1 { charsArg = args[1] } var chars []byte switch { case charsArg.isInstance(UnicodeType): u, raised := s.Decode(f, EncodeDefault, EncodeStrict) if raised != nil { return nil, raised } return unicodeStrip(f, Args{u.ToObject(), charsArg}, nil) case charsArg.isInstance(StrType): chars = []byte(toStrUnsafe(charsArg).Value()) case charsArg == None: chars = strASCIISpaces default: return nil, f.RaiseType(TypeErrorType, "strip arg must be None, str or unicode") } byteSlice := []byte(s.Value()) numBytes := len(byteSlice) lindex := 0 if side == stripSideLeft || side == stripSideBoth { LeftStrip: for ; lindex < numBytes; lindex++ { b := byteSlice[lindex] for _, c := range chars { if b == c { continue LeftStrip } } break } } rindex := numBytes if side == stripSideRight || side == stripSideBoth { RightStrip: for ; rindex > lindex; rindex-- { b := byteSlice[rindex-1] for _, c := range chars { if b == c { continue RightStrip } } break } } return NewStr(string(byteSlice[lindex:rindex])).ToObject(), nil } func strStartsWith(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return strStartsEndsWith(f, "startswith", args) } func strStr(_ *Frame, o *Object) (*Object, *BaseException) { if o.typ == StrType { return o, nil } return NewStr(toStrUnsafe(o).Value()).ToObject(), nil } func strSwapCase(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "swapcase", args, StrType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() numBytes := len(s) if numBytes == 0 { return args[0], nil } b := make([]byte, numBytes) for i := 0; i < numBytes; i++ { if isLower(s[i]) { b[i] = toUpper(s[i]) } else if isUpper(s[i]) { b[i] = toLower(s[i]) } else { b[i] = s[i] } } return NewStr(string(b)).ToObject(), nil } func initStrType(dict map[string]*Object) { dict["__getnewargs__"] = newBuiltinFunction("__getnewargs__", strGetNewArgs).ToObject() dict["capitalize"] = newBuiltinFunction("capitalize", strCapitalize).ToObject() dict["count"] = newBuiltinFunction("count", strCount).ToObject() dict["center"] = newBuiltinFunction("center", strCenter).ToObject() dict["decode"] = newBuiltinFunction("decode", strDecode).ToObject() dict["endswith"] = newBuiltinFunction("endswith", strEndsWith).ToObject() dict["find"] = newBuiltinFunction("find", strFind).ToObject() dict["index"] = newBuiltinFunction("index", strIndex).ToObject() dict["isalnum"] = newBuiltinFunction("isalnum", strIsAlNum).ToObject() dict["isalpha"] = newBuiltinFunction("isalpha", strIsAlpha).ToObject() dict["isdigit"] = newBuiltinFunction("isdigit", strIsDigit).ToObject() dict["islower"] = newBuiltinFunction("islower", strIsLower).ToObject() dict["isspace"] = newBuiltinFunction("isspace", strIsSpace).ToObject() dict["istitle"] = newBuiltinFunction("istitle", strIsTitle).ToObject() dict["isupper"] = newBuiltinFunction("isupper", strIsUpper).ToObject() dict["join"] = newBuiltinFunction("join", strJoin).ToObject() dict["lower"] = newBuiltinFunction("lower", strLower).ToObject() dict["ljust"] = newBuiltinFunction("ljust", strLJust).ToObject() dict["lstrip"] = newBuiltinFunction("lstrip", strLStrip).ToObject() dict["rfind"] = newBuiltinFunction("rfind", strRFind).ToObject() dict["rindex"] = newBuiltinFunction("rindex", strRIndex).ToObject() dict["rjust"] = newBuiltinFunction("rjust", strRJust).ToObject() dict["split"] = newBuiltinFunction("split", strSplit).ToObject() dict["splitlines"] = newBuiltinFunction("splitlines", strSplitLines).ToObject() dict["startswith"] = newBuiltinFunction("startswith", strStartsWith).ToObject() dict["strip"] = newBuiltinFunction("strip", strStrip).ToObject() dict["swapcase"] = newBuiltinFunction("swapcase", strSwapCase).ToObject() dict["replace"] = newBuiltinFunction("replace", strReplace).ToObject() dict["rstrip"] = newBuiltinFunction("rstrip", strRStrip).ToObject() dict["title"] = newBuiltinFunction("title", strTitle).ToObject() dict["upper"] = newBuiltinFunction("upper", strUpper).ToObject() dict["zfill"] = newBuiltinFunction("zfill", strZFill).ToObject() StrType.slots.Add = &binaryOpSlot{strAdd} StrType.slots.Contains = &binaryOpSlot{strContains} StrType.slots.Eq = &binaryOpSlot{strEq} StrType.slots.GE = &binaryOpSlot{strGE} StrType.slots.GetItem = &binaryOpSlot{strGetItem} StrType.slots.GT = &binaryOpSlot{strGT} StrType.slots.Hash = &unaryOpSlot{strHash} StrType.slots.LE = &binaryOpSlot{strLE} StrType.slots.Len = &unaryOpSlot{strLen} StrType.slots.LT = &binaryOpSlot{strLT} StrType.slots.Mod = &binaryOpSlot{strMod} StrType.slots.Mul = &binaryOpSlot{strMul} StrType.slots.NE = &binaryOpSlot{strNE} StrType.slots.New = &newSlot{strNew} StrType.slots.Native = &nativeSlot{strNative} StrType.slots.Repr = &unaryOpSlot{strRepr} StrType.slots.RMul = &binaryOpSlot{strMul} StrType.slots.Str = &unaryOpSlot{strStr} } func strCompare(v, w *Object, ltResult, eqResult, gtResult *Int) *Object { if v == w { return eqResult.ToObject() } if !w.isInstance(StrType) { return NotImplemented } s1 := toStrUnsafe(v).Value() s2 := toStrUnsafe(w).Value() if s1 < s2 { return ltResult.ToObject() } if s1 == s2 { return eqResult.ToObject() } return gtResult.ToObject() } func strInterpolate(f *Frame, format string, values *Tuple) (*Object, *BaseException) { var buf bytes.Buffer valueIndex := 0 index := strings.Index(format, "%") for index != -1 { buf.WriteString(format[:index]) format = format[index:] matches := strInterpolationRegexp.FindStringSubmatch(format) if matches == nil { return nil, f.RaiseType(ValueErrorType, "invalid format spec") } flags, fieldType := matches[1], matches[7] if fieldType != "%" && valueIndex >= len(values.elems) { return nil, f.RaiseType(TypeErrorType, "not enough arguments for format string") } fieldWidth := -1 if matches[2] == "*" || matches[4] != "" { return nil, f.RaiseType(NotImplementedErrorType, "field width not yet supported") } if matches[2] != "" { var err error fieldWidth, err = strconv.Atoi(matches[2]) if err != nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprint(err)) } } if flags != "" && flags != "0" { return nil, f.RaiseType(NotImplementedErrorType, "conversion flags not yet supported") } var val string switch fieldType { case "r", "s": o := values.elems[valueIndex] var s *Str var raised *BaseException if fieldType == "r" { s, raised = Repr(f, o) } else { s, raised = ToStr(f, o) } if raised != nil { return nil, raised } val = s.Value() if fieldWidth > 0 { val = strLeftPad(val, fieldWidth, " ") } buf.WriteString(val) valueIndex++ case "f": o := values.elems[valueIndex] if v, ok := floatCoerce(o); ok { val := strconv.FormatFloat(v, 'f', 6, 64) if fieldWidth > 0 { fillchar := " " if flags != "" { fillchar = flags } val = strLeftPad(val, fieldWidth, fillchar) } buf.WriteString(val) valueIndex++ } else { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("float argument required, not %s", o.typ.Name())) } case "d", "x", "X", "o": o := values.elems[valueIndex] i, raised := ToInt(f, values.elems[valueIndex]) if raised != nil { return nil, raised } if fieldType == "d" { s, raised := ToStr(f, i) if raised != nil { return nil, raised } val = s.Value() } else if matches[7] == "o" { if o.isInstance(LongType) { val = toLongUnsafe(o).Value().Text(8) } else { val = strconv.FormatInt(int64(toIntUnsafe(i).Value()), 8) } } else { if o.isInstance(LongType) { val = toLongUnsafe(o).Value().Text(16) } else { val = strconv.FormatInt(int64(toIntUnsafe(i).Value()), 16) } if fieldType == "X" { val = strings.ToUpper(val) } } if fieldWidth > 0 { fillchar := " " if flags != "" { fillchar = flags } val = strLeftPad(val, fieldWidth, fillchar) } buf.WriteString(val) valueIndex++ case "%": val = "%" if fieldWidth > 0 { val = strLeftPad(val, fieldWidth, " ") } buf.WriteString(val) default: format := "conversion type not yet supported: %s" return nil, f.RaiseType(NotImplementedErrorType, fmt.Sprintf(format, fieldType)) } format = format[len(matches[0]):] index = strings.Index(format, "%") } if valueIndex < len(values.elems) { return nil, f.RaiseType(TypeErrorType, "not all arguments converted during string formatting") } buf.WriteString(format) return NewStr(buf.String()).ToObject(), nil } func strRepeatCount(f *Frame, numChars int, mult *Object) (int, bool, *BaseException) { var n int switch { case mult.isInstance(IntType): n = toIntUnsafe(mult).Value() case mult.isInstance(LongType): l := toLongUnsafe(mult).Value() if !numInIntRange(l) { return 0, false, f.RaiseType(OverflowErrorType, fmt.Sprintf("cannot fit '%s' into an index-sized integer", mult.typ.Name())) } n = int(l.Int64()) default: return 0, false, nil } if n <= 0 { return 0, true, nil } if numChars > MaxInt/n { return 0, false, f.RaiseType(OverflowErrorType, errResultTooLarge) } return n, true, nil } func adjustIndex(start, end, length int) (int, int) { if end > length { end = length } else if end < 0 { end += length if end < 0 { end = 0 } } if start < 0 { start += length if start < 0 { start = 0 } } return start, end } func strStartsEndsWith(f *Frame, method string, args Args) (*Object, *BaseException) { expectedTypes := []*Type{StrType, ObjectType, IntType, IntType} argc := len(args) if argc == 2 || argc == 3 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, method, args, expectedTypes...); raised != nil { return nil, raised } matchesArg := args[1] var matches []string switch { case matchesArg.isInstance(TupleType): elems := toTupleUnsafe(matchesArg).elems matches = make([]string, len(elems)) for i, o := range elems { if !o.isInstance(BaseStringType) { return nil, f.RaiseType(TypeErrorType, "expected a str") } s, raised := ToStr(f, o) if raised != nil { return nil, raised } matches[i] = s.Value() } case matchesArg.isInstance(BaseStringType): s, raised := ToStr(f, matchesArg) if raised != nil { return nil, raised } matches = []string{s.Value()} default: msg := " first arg must be str, unicode, or tuple, not " return nil, f.RaiseType(TypeErrorType, method+msg+matchesArg.typ.Name()) } s := toStrUnsafe(args[0]).Value() l := len(s) start, end := 0, l if argc >= 3 { start = toIntUnsafe(args[2]).Value() } if argc == 4 { end = toIntUnsafe(args[3]).Value() } start, end = adjustIndex(start, end, l) if start > end { // start == end may still return true when matching ''. return False.ToObject(), nil } s = s[start:end] matcher := strings.HasPrefix if method == "endswith" { matcher = strings.HasSuffix } for _, match := range matches { if matcher(s, match) { return True.ToObject(), nil } } return False.ToObject(), nil } func strTitle(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{StrType} if raised := checkMethodArgs(f, "title", args, expectedTypes...); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() numBytes := len(s) if numBytes == 0 { return args[0], nil } b := make([]byte, numBytes) previousIsCased := false for i := 0; i < numBytes; i++ { c := s[i] switch { case isLower(c): if !previousIsCased { c = toUpper(c) } previousIsCased = true case isUpper(c): if previousIsCased { c = toLower(c) } previousIsCased = true default: previousIsCased = false } b[i] = c } return NewStr(string(b)).ToObject(), nil } func strUpper(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{StrType} if raised := checkMethodArgs(f, "upper", args, expectedTypes...); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() numBytes := len(s) if numBytes == 0 { return args[0], nil } b := make([]byte, numBytes) for i := 0; i < numBytes; i++ { b[i] = toUpper(s[i]) } return NewStr(string(b)).ToObject(), nil } func strZFill(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "zfill", args, StrType, ObjectType); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() width, raised := ToIntValue(f, args[1]) if raised != nil { return nil, raised } return NewStr(strLeftPad(s, width, "0")).ToObject(), nil } func init() { InternStr("") for i := 0; i < 256; i++ { InternStr(string([]byte{byte(i)})) } } func toLower(b byte) byte { if isUpper(b) { return b + caseOffset } return b } func toUpper(b byte) byte { if isLower(b) { return b - caseOffset } return b } func isAlNum(c byte) bool { return isAlpha(c) || isDigit(c) } func isAlpha(c byte) bool { return isUpper(c) || isLower(c) } func isDigit(c byte) bool { return '0' <= c && c <= '9' } func isLower(c byte) bool { return 'a' <= c && c <= 'z' } func isSpace(c byte) bool { switch c { case ' ', '\n', '\t', '\v', '\f', '\r': return true default: return false } } func isUpper(c byte) bool { return 'A' <= c && c <= 'Z' } func pad(s string, left int, right int, fillchar string) string { buf := bytes.Buffer{} if left < 0 { left = 0 } if right < 0 { right = 0 } if left == 0 && right == 0 { return s } buf.Grow(left + len(s) + right) buf.WriteString(strings.Repeat(fillchar, left)) buf.WriteString(s) buf.WriteString(strings.Repeat(fillchar, right)) return buf.String() } // strLeftPad returns s padded with fillchar so that its length is at least width. // Fillchar must be a single character. When fillchar is "0", s starting with a // sign are handled correctly. func strLeftPad(s string, width int, fillchar string) string { l := len(s) if width <= l { return s } buf := bytes.Buffer{} buf.Grow(width) if l > 0 && fillchar == "0" && (s[0] == '-' || s[0] == '+') { buf.WriteByte(s[0]) s = s[1:] l = len(s) width-- } // TODO: Support or throw fillchar len more than one. buf.WriteString(strings.Repeat(fillchar, width-l)) buf.WriteString(s) return buf.String() } type indexFunc func(string, string) (int, *BaseException) func strFindOrIndex(f *Frame, args Args, fn indexFunc) (*Object, *BaseException) { // TODO: Support for unicode substring. expectedTypes := []*Type{StrType, StrType, ObjectType, ObjectType} argc := len(args) if argc == 2 || argc == 3 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "find/index", args, expectedTypes...); raised != nil { return nil, raised } s := toStrUnsafe(args[0]).Value() l := len(s) start, end := 0, l var raised *BaseException if argc >= 3 && args[2] != None { start, raised = IndexInt(f, args[2]) if raised != nil { return nil, raised } } if argc == 4 && args[3] != None { end, raised = IndexInt(f, args[3]) if raised != nil { return nil, raised } } // Default to an impossible search. search, sub := "", "-" if start <= l { start, end = adjustIndex(start, end, l) if start <= end { sub = toStrUnsafe(args[1]).Value() search = s[start:end] } } index, raised := fn(search, sub) if raised != nil { return nil, raised } if index != -1 { index += start } return NewInt(index).ToObject(), nil } func strJustDecodeArgs(f *Frame, args Args, name string) (string, int, string, *BaseException) { expectedTypes := []*Type{StrType, IntType, StrType} if raised := checkMethodArgs(f, name, args, expectedTypes...); raised != nil { return "", 0, "", raised } s := toStrUnsafe(args[0]).Value() width := toIntUnsafe(args[1]).Value() fill := toStrUnsafe(args[2]).Value() if numChars := len(fill); numChars != 1 { return s, width, fill, f.RaiseType(TypeErrorType, fmt.Sprintf("%[1]s() argument 2 must be char, not str", name)) } return s, width, fill, nil } ================================================ FILE: runtime/str_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "math/big" "reflect" "runtime" "testing" ) func TestNewStr(t *testing.T) { expected := &Str{Object: Object{typ: StrType}, value: "foo"} s := NewStr("foo") if !reflect.DeepEqual(s, expected) { t.Errorf(`NewStr("foo") = %+v, expected %+v`, *s, *expected) } } func BenchmarkNewStr(b *testing.B) { var ret *Str for i := 0; i < b.N; i++ { ret = NewStr("foo") } runtime.KeepAlive(ret) } // # On a 64bit system: // >>> hash("foo") // -4177197833195190597 // >>> hash("bar") // 327024216814240868 // >>> hash("baz") // 327024216814240876 func TestHashString(t *testing.T) { truncateInt := func(i int64) int { return int(i) } // Support for 32bit platforms cases := []struct { value string hash int }{ {"foo", truncateInt(-4177197833195190597)}, {"bar", truncateInt(327024216814240868)}, {"baz", truncateInt(327024216814240876)}, } for _, cas := range cases { if h := hashString(cas.value); h != cas.hash { t.Errorf("hashString(%q) = %d, expected %d", cas.value, h, cas.hash) } } } func TestStrBinaryOps(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, fn binaryOpFunc, v *Object, w *Object) (*Object, *BaseException) { return fn(f, v, w) }) cases := []invokeTestCase{ {args: wrapArgs(Add, "foo", "bar"), want: NewStr("foobar").ToObject()}, {args: wrapArgs(Add, "foo", NewUnicode("bar")), want: NewUnicode("foobar").ToObject()}, {args: wrapArgs(Add, "baz", ""), want: NewStr("baz").ToObject()}, {args: wrapArgs(Add, "", newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'str' and 'object'")}, {args: wrapArgs(Add, None, ""), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'NoneType' and 'str'")}, {args: wrapArgs(Mod, "%s", 42), want: NewStr("42").ToObject()}, {args: wrapArgs(Mod, "%3s", 42), want: NewStr(" 42").ToObject()}, {args: wrapArgs(Mod, "%03s", 42), want: NewStr(" 42").ToObject()}, {args: wrapArgs(Mod, "%f", 3.14), want: NewStr("3.140000").ToObject()}, {args: wrapArgs(Mod, "%10f", 3.14), want: NewStr(" 3.140000").ToObject()}, {args: wrapArgs(Mod, "%010f", 3.14), want: NewStr("003.140000").ToObject()}, {args: wrapArgs(Mod, "abc %d", NewLong(big.NewInt(123))), want: NewStr("abc 123").ToObject()}, {args: wrapArgs(Mod, "%d", 3.14), want: NewStr("3").ToObject()}, {args: wrapArgs(Mod, "%%", NewTuple()), want: NewStr("%").ToObject()}, {args: wrapArgs(Mod, "%3%", NewTuple()), want: NewStr(" %").ToObject()}, {args: wrapArgs(Mod, "%03%", NewTuple()), want: NewStr(" %").ToObject()}, {args: wrapArgs(Mod, "%r", "abc"), want: NewStr("'abc'").ToObject()}, {args: wrapArgs(Mod, "%6r", "abc"), want: NewStr(" 'abc'").ToObject()}, {args: wrapArgs(Mod, "%06r", "abc"), want: NewStr(" 'abc'").ToObject()}, {args: wrapArgs(Mod, "%s %s", true), wantExc: mustCreateException(TypeErrorType, "not enough arguments for format string")}, {args: wrapArgs(Mod, "%Z", None), wantExc: mustCreateException(ValueErrorType, "invalid format spec")}, {args: wrapArgs(Mod, "%s", NewDict()), wantExc: mustCreateException(NotImplementedErrorType, "mappings not yet supported")}, {args: wrapArgs(Mod, "% d", 23), wantExc: mustCreateException(NotImplementedErrorType, "conversion flags not yet supported")}, {args: wrapArgs(Mod, "%.3f", 102.1), wantExc: mustCreateException(NotImplementedErrorType, "field width not yet supported")}, {args: wrapArgs(Mod, "%x", 0x1f), want: NewStr("1f").ToObject()}, {args: wrapArgs(Mod, "%X", 0xffff), want: NewStr("FFFF").ToObject()}, {args: wrapArgs(Mod, "%x", 1.2), want: NewStr("1").ToObject()}, {args: wrapArgs(Mod, "abc %x", NewLong(big.NewInt(123))), want: NewStr("abc 7b").ToObject()}, {args: wrapArgs(Mod, "%x", None), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, {args: wrapArgs(Mod, "%f", None), wantExc: mustCreateException(TypeErrorType, "float argument required, not NoneType")}, {args: wrapArgs(Mod, "%s", newTestTuple(123, None)), wantExc: mustCreateException(TypeErrorType, "not all arguments converted during string formatting")}, {args: wrapArgs(Mod, "%d", newTestTuple("123")), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, {args: wrapArgs(Mod, "%o", newTestTuple(123)), want: NewStr("173").ToObject()}, {args: wrapArgs(Mod, "%o", 8), want: NewStr("10").ToObject()}, {args: wrapArgs(Mod, "%o", -8), want: NewStr("-10").ToObject()}, {args: wrapArgs(Mod, "%03o", newTestTuple(123)), want: NewStr("173").ToObject()}, {args: wrapArgs(Mod, "%04o", newTestTuple(123)), want: NewStr("0173").ToObject()}, {args: wrapArgs(Mod, "%o", newTestTuple("123")), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, {args: wrapArgs(Mod, "%o", None), wantExc: mustCreateException(TypeErrorType, "an integer is required")}, {args: wrapArgs(Mul, "", 10), want: NewStr("").ToObject()}, {args: wrapArgs(Mul, "foo", -2), want: NewStr("").ToObject()}, {args: wrapArgs(Mul, "foobar", 0), want: NewStr("").ToObject()}, {args: wrapArgs(Mul, "aloha", 2), want: NewStr("alohaaloha").ToObject()}, {args: wrapArgs(Mul, 1, "baz"), want: NewStr("baz").ToObject()}, {args: wrapArgs(Mul, newObject(ObjectType), "qux"), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'object' and 'str'")}, {args: wrapArgs(Mul, "foo", ""), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'str' and 'str'")}, {args: wrapArgs(Mul, "bar", MaxInt), wantExc: mustCreateException(OverflowErrorType, "result too large")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestStrCompare(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs("", ""), want: compareAllResultEq}, {args: wrapArgs("foo", "foo"), want: compareAllResultEq}, {args: wrapArgs("", "foo"), want: compareAllResultLT}, {args: wrapArgs("foo", ""), want: compareAllResultGT}, {args: wrapArgs("bar", "baz"), want: compareAllResultLT}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestStrContains(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs("foobar", "foo"), want: True.ToObject()}, {args: wrapArgs("abcdef", "bar"), want: False.ToObject()}, {args: wrapArgs("", ""), want: True.ToObject()}, {args: wrapArgs("foobar", NewUnicode("bar")), want: True.ToObject()}, {args: wrapArgs("", 102.1), wantExc: mustCreateException(TypeErrorType, "'in ' requires string as left operand, not float")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(StrType, "__contains__", &cas); err != "" { t.Error(err) } } } func TestStrDecode(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs("foo"), want: NewUnicode("foo").ToObject()}, {args: wrapArgs("foo\xffbar", "utf8", "replace"), want: NewUnicode("foo\ufffdbar").ToObject()}, {args: wrapArgs("foo\xffbar", "utf8", "ignore"), want: NewUnicode("foobar").ToObject()}, // Bad error handler name only triggers LookupError when an // error is encountered. {args: wrapArgs("foobar", "utf8", "noexist"), want: NewUnicode("foobar").ToObject()}, {args: wrapArgs("foo\xffbar", "utf8", "noexist"), wantExc: mustCreateException(LookupErrorType, "unknown error handler name 'noexist'")}, {args: wrapArgs("foobar", "noexist"), wantExc: mustCreateException(LookupErrorType, "unknown encoding: noexist")}, {args: wrapArgs("foo\xffbar"), wantExc: mustCreateException(UnicodeDecodeErrorType, "'utf8' codec can't decode byte 0xff in position 3")}, // Surrogates are not valid UTF-8 and should raise, unlike // CPython 2.x. {args: wrapArgs("foo\xef\xbf\xbdbar", "utf8", "strict"), wantExc: mustCreateException(UnicodeDecodeErrorType, "'utf8' codec can't decode byte 0xef in position 3")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(StrType, "decode", &cas); err != "" { t.Error(err) } } } func TestStrGetItem(t *testing.T) { intIndexType := newTestClass("IntIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(2).ToObject(), nil }).ToObject(), })) longIndexType := newTestClass("LongIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewLong(big.NewInt(2)).ToObject(), nil }).ToObject(), })) cases := []invokeTestCase{ {args: wrapArgs("bar", 1), want: NewStr("a").ToObject()}, {args: wrapArgs("foo", 3.14), wantExc: mustCreateException(TypeErrorType, "string indices must be integers or slice, not float")}, {args: wrapArgs("bar", big.NewInt(1)), want: NewStr("a").ToObject()}, {args: wrapArgs("baz", -1), want: NewStr("z").ToObject()}, {args: wrapArgs("baz", newObject(intIndexType)), want: NewStr("z").ToObject()}, {args: wrapArgs("baz", newObject(longIndexType)), want: NewStr("z").ToObject()}, {args: wrapArgs("baz", -4), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs("", 0), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs("foo", 3), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs("bar", newTestSlice(None, 2)), want: NewStr("ba").ToObject()}, {args: wrapArgs("bar", newTestSlice(1, 3)), want: NewStr("ar").ToObject()}, {args: wrapArgs("bar", newTestSlice(1, None)), want: NewStr("ar").ToObject()}, {args: wrapArgs("foobarbaz", newTestSlice(1, 8, 2)), want: NewStr("obra").ToObject()}, {args: wrapArgs("abc", newTestSlice(None, None, -1)), want: NewStr("cba").ToObject()}, {args: wrapArgs("bar", newTestSlice(1, 2, 0)), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(StrType, "__getitem__", &cas); err != "" { t.Error(err) } } } func TestStrNew(t *testing.T) { dummy := newObject(ObjectType) dummyStr := NewStr(fmt.Sprintf("", dummy)) fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__str__": newBuiltinFunction("__str__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewInt(123).ToObject(), nil }).ToObject(), })) foo := newObject(fooType) strictEqType := newTestClassStrictEq("StrictEq", StrType) subType := newTestClass("SubType", []*Type{StrType}, newStringDict(map[string]*Object{})) subTypeObject := (&Str{Object: Object{typ: subType}, value: "abc"}).ToObject() goodSlotType := newTestClass("GoodSlot", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__str__": newBuiltinFunction("__str__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewStr("abc").ToObject(), nil }).ToObject(), })) badSlotType := newTestClass("BadSlot", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__str__": newBuiltinFunction("__str__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return newObject(ObjectType), nil }).ToObject(), })) slotSubTypeType := newTestClass("SlotSubType", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__str__": newBuiltinFunction("__str__", func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return subTypeObject, nil }).ToObject(), })) cases := []invokeTestCase{ {wantExc: mustCreateException(TypeErrorType, "'__new__' requires 1 arguments")}, {args: wrapArgs(IntType.ToObject()), wantExc: mustCreateException(TypeErrorType, "str.__new__(int): int is not a subtype of str")}, {args: wrapArgs(StrType.ToObject(), NewInt(1).ToObject(), NewInt(2).ToObject()), wantExc: mustCreateException(TypeErrorType, "str() takes at most 1 argument (2 given)")}, {args: wrapArgs(StrType.ToObject(), foo), wantExc: mustCreateException(TypeErrorType, "__str__ returned non-string (type int)")}, {args: wrapArgs(StrType.ToObject()), want: NewStr("").ToObject()}, {args: wrapArgs(StrType.ToObject(), NewDict().ToObject()), want: NewStr("{}").ToObject()}, {args: wrapArgs(StrType.ToObject(), dummy), want: dummyStr.ToObject()}, {args: wrapArgs(strictEqType, "foo"), want: (&Str{Object: Object{typ: strictEqType}, value: "foo"}).ToObject()}, {args: wrapArgs(StrType, newObject(goodSlotType)), want: NewStr("abc").ToObject()}, {args: wrapArgs(StrType, newObject(badSlotType)), wantExc: mustCreateException(TypeErrorType, "__str__ returned non-string (type object)")}, {args: wrapArgs(StrType, newObject(slotSubTypeType)), want: subTypeObject}, {args: wrapArgs(strictEqType, newObject(goodSlotType)), want: (&Str{Object: Object{typ: strictEqType}, value: "abc"}).ToObject()}, {args: wrapArgs(strictEqType, newObject(badSlotType)), wantExc: mustCreateException(TypeErrorType, "__str__ returned non-string (type object)")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(StrType, "__new__", &cas); err != "" { t.Error(err) } } } func TestStrRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs("foo"), want: NewStr(`'foo'`).ToObject()}, {args: wrapArgs("on\nmultiple\nlines"), want: NewStr(`'on\nmultiple\nlines'`).ToObject()}, {args: wrapArgs("\x00\x00"), want: NewStr(`'\x00\x00'`).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(StrType, "__repr__", &cas); err != "" { t.Error(err) } } } func TestStrMethods(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{"bar": None})) intIndexType := newTestClass("IntIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(2).ToObject(), nil }).ToObject(), })) longIndexType := newTestClass("LongIndex", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__index__": newBuiltinFunction("__index__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewLong(big.NewInt(2)).ToObject(), nil }).ToObject(), })) intIntType := newTestClass("IntInt", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__int__": newBuiltinFunction("__int__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewInt(2).ToObject(), nil }).ToObject(), })) longIntType := newTestClass("LongInt", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__int__": newBuiltinFunction("__int__", func(f *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return NewLong(big.NewInt(2)).ToObject(), nil }).ToObject(), })) cases := []struct { methodName string args Args want *Object wantExc *BaseException }{ {"capitalize", wrapArgs(""), NewStr("").ToObject(), nil}, {"capitalize", wrapArgs("foobar"), NewStr("Foobar").ToObject(), nil}, {"capitalize", wrapArgs("FOOBAR"), NewStr("Foobar").ToObject(), nil}, {"capitalize", wrapArgs("ùBAR"), NewStr("ùbar").ToObject(), nil}, {"capitalize", wrapArgs("вол"), NewStr("вол").ToObject(), nil}, {"capitalize", wrapArgs("foobar", 123), nil, mustCreateException(TypeErrorType, "'capitalize' of 'str' requires 1 arguments")}, {"capitalize", wrapArgs("ВОЛ"), NewStr("ВОЛ").ToObject(), nil}, {"center", wrapArgs("foobar", 9, "#"), NewStr("##foobar#").ToObject(), nil}, {"center", wrapArgs("foobar", 10, "#"), NewStr("##foobar##").ToObject(), nil}, {"center", wrapArgs("foobar", 3, "#"), NewStr("foobar").ToObject(), nil}, {"center", wrapArgs("foobar", -1, "#"), NewStr("foobar").ToObject(), nil}, {"center", wrapArgs("foobar", 10, "##"), nil, mustCreateException(TypeErrorType, "center() argument 2 must be char, not str")}, {"center", wrapArgs("foobar", 10, ""), nil, mustCreateException(TypeErrorType, "center() argument 2 must be char, not str")}, {"count", wrapArgs("", "a"), NewInt(0).ToObject(), nil}, {"count", wrapArgs("five", ""), NewInt(5).ToObject(), nil}, {"count", wrapArgs("abba", "bb"), NewInt(1).ToObject(), nil}, {"count", wrapArgs("abbba", "bb"), NewInt(1).ToObject(), nil}, {"count", wrapArgs("abbbba", "bb"), NewInt(2).ToObject(), nil}, {"count", wrapArgs("abcdeffdeabcb", "b"), NewInt(3).ToObject(), nil}, {"count", wrapArgs(""), nil, mustCreateException(TypeErrorType, "'count' of 'str' requires 2 arguments")}, {"endswith", wrapArgs("", ""), True.ToObject(), nil}, {"endswith", wrapArgs("", "", 1), False.ToObject(), nil}, {"endswith", wrapArgs("foobar", "bar"), True.ToObject(), nil}, {"endswith", wrapArgs("foobar", "bar", 0, -2), False.ToObject(), nil}, {"endswith", wrapArgs("foobar", "foo", 0, 3), True.ToObject(), nil}, {"endswith", wrapArgs("foobar", "bar", 3, 5), False.ToObject(), nil}, {"endswith", wrapArgs("foobar", "bar", 5, 3), False.ToObject(), nil}, {"endswith", wrapArgs("bar", "foobar"), False.ToObject(), nil}, {"endswith", wrapArgs("foo", newTestTuple("barfoo", "oo").ToObject()), True.ToObject(), nil}, {"endswith", wrapArgs("foo", 123), nil, mustCreateException(TypeErrorType, "endswith first arg must be str, unicode, or tuple, not int")}, {"endswith", wrapArgs("foo", newTestTuple(123).ToObject()), nil, mustCreateException(TypeErrorType, "expected a str")}, {"find", wrapArgs("", ""), NewInt(0).ToObject(), nil}, {"find", wrapArgs("", "", 1), NewInt(-1).ToObject(), nil}, {"find", wrapArgs("", "", -1), NewInt(0).ToObject(), nil}, {"find", wrapArgs("", "", None, -1), NewInt(0).ToObject(), nil}, {"find", wrapArgs("foobar", "bar"), NewInt(3).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", fooType), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"find", wrapArgs("foobar", "bar", NewInt(MaxInt)), NewInt(-1).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", None, NewInt(MaxInt)), NewInt(3).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", newObject(intIndexType)), NewInt(3).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", None, newObject(intIndexType)), NewInt(-1).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", newObject(longIndexType)), NewInt(3).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", None, newObject(longIndexType)), NewInt(-1).ToObject(), nil}, // TODO: Support unicode substring. {"find", wrapArgs("foobar", NewUnicode("bar")), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'unicode'")}, {"find", wrapArgs("foobar", "bar", "baz"), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"find", wrapArgs("foobar", "bar", 0, "baz"), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"find", wrapArgs("foobar", "bar", None), NewInt(3).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", 0, None), NewInt(3).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", 0, -2), NewInt(-1).ToObject(), nil}, {"find", wrapArgs("foobar", "foo", 0, 3), NewInt(0).ToObject(), nil}, {"find", wrapArgs("foobar", "foo", 10), NewInt(-1).ToObject(), nil}, {"find", wrapArgs("foobar", "foo", 3, 3), NewInt(-1).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", 3, 5), NewInt(-1).ToObject(), nil}, {"find", wrapArgs("foobar", "bar", 5, 3), NewInt(-1).ToObject(), nil}, {"find", wrapArgs("bar", "foobar"), NewInt(-1).ToObject(), nil}, {"find", wrapArgs("bar", "a", 1, 10), NewInt(1).ToObject(), nil}, {"find", wrapArgs("bar", "a", NewLong(big.NewInt(1)), 10), NewInt(1).ToObject(), nil}, {"find", wrapArgs("bar", "a", 0, NewLong(big.NewInt(2))), NewInt(1).ToObject(), nil}, {"find", wrapArgs("bar", "a", 1, 3), NewInt(1).ToObject(), nil}, {"find", wrapArgs("bar", "a", 0, -1), NewInt(1).ToObject(), nil}, {"find", wrapArgs("foo", newTestTuple("barfoo", "oo").ToObject()), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'tuple'")}, {"find", wrapArgs("foo", 123), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'int'")}, {"index", wrapArgs("", ""), NewInt(0).ToObject(), nil}, {"index", wrapArgs("", "", 1), nil, mustCreateException(ValueErrorType, "substring not found")}, {"index", wrapArgs("", "", -1), NewInt(0).ToObject(), nil}, {"index", wrapArgs("", "", None, -1), NewInt(0).ToObject(), nil}, {"index", wrapArgs("foobar", "bar"), NewInt(3).ToObject(), nil}, {"index", wrapArgs("foobar", "bar", fooType), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"index", wrapArgs("foobar", "bar", NewInt(MaxInt)), nil, mustCreateException(ValueErrorType, "substring not found")}, {"index", wrapArgs("foobar", "bar", None, NewInt(MaxInt)), NewInt(3).ToObject(), nil}, {"index", wrapArgs("foobar", "bar", newObject(intIndexType)), NewInt(3).ToObject(), nil}, {"index", wrapArgs("foobar", "bar", None, newObject(intIndexType)), nil, mustCreateException(ValueErrorType, "substring not found")}, {"index", wrapArgs("foobar", "bar", newObject(longIndexType)), NewInt(3).ToObject(), nil}, {"index", wrapArgs("foobar", "bar", None, newObject(longIndexType)), nil, mustCreateException(ValueErrorType, "substring not found")}, //TODO: Support unicode substring. {"index", wrapArgs("foobar", NewUnicode("bar")), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'unicode'")}, {"index", wrapArgs("foobar", "bar", "baz"), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"index", wrapArgs("foobar", "bar", 0, "baz"), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"index", wrapArgs("foobar", "bar", None), NewInt(3).ToObject(), nil}, {"index", wrapArgs("foobar", "bar", 0, None), NewInt(3).ToObject(), nil}, {"index", wrapArgs("foobar", "bar", 0, -2), nil, mustCreateException(ValueErrorType, "substring not found")}, {"index", wrapArgs("foobar", "foo", 0, 3), NewInt(0).ToObject(), nil}, {"index", wrapArgs("foobar", "foo", 10), nil, mustCreateException(ValueErrorType, "substring not found")}, {"index", wrapArgs("foobar", "foo", 3, 3), nil, mustCreateException(ValueErrorType, "substring not found")}, {"index", wrapArgs("foobar", "bar", 3, 5), nil, mustCreateException(ValueErrorType, "substring not found")}, {"index", wrapArgs("foobar", "bar", 5, 3), nil, mustCreateException(ValueErrorType, "substring not found")}, {"index", wrapArgs("bar", "foobar"), nil, mustCreateException(ValueErrorType, "substring not found")}, {"index", wrapArgs("bar", "a", 1, 10), NewInt(1).ToObject(), nil}, {"index", wrapArgs("bar", "a", NewLong(big.NewInt(1)), 10), NewInt(1).ToObject(), nil}, {"index", wrapArgs("bar", "a", 0, NewLong(big.NewInt(2))), NewInt(1).ToObject(), nil}, {"index", wrapArgs("bar", "a", 1, 3), NewInt(1).ToObject(), nil}, {"index", wrapArgs("bar", "a", 0, -1), NewInt(1).ToObject(), nil}, {"index", wrapArgs("foo", newTestTuple("barfoo", "oo").ToObject()), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'tuple'")}, {"index", wrapArgs("foo", 123), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'int'")}, {"index", wrapArgs("barbaz", "ba"), NewInt(0).ToObject(), nil}, {"index", wrapArgs("barbaz", "ba", 1), NewInt(3).ToObject(), nil}, {"isalnum", wrapArgs("123abc"), True.ToObject(), nil}, {"isalnum", wrapArgs(""), False.ToObject(), nil}, {"isalnum", wrapArgs("#$%"), False.ToObject(), nil}, {"isalnum", wrapArgs("abc#123"), False.ToObject(), nil}, {"isalnum", wrapArgs("123abc", "efg"), nil, mustCreateException(TypeErrorType, "'isalnum' of 'str' requires 1 arguments")}, {"isalpha", wrapArgs("xyz"), True.ToObject(), nil}, {"isalpha", wrapArgs(""), False.ToObject(), nil}, {"isalpha", wrapArgs("#$%"), False.ToObject(), nil}, {"isalpha", wrapArgs("abc#123"), False.ToObject(), nil}, {"isalpha", wrapArgs("absd", "efg"), nil, mustCreateException(TypeErrorType, "'isalpha' of 'str' requires 1 arguments")}, {"isdigit", wrapArgs("abc"), False.ToObject(), nil}, {"isdigit", wrapArgs("123"), True.ToObject(), nil}, {"isdigit", wrapArgs(""), False.ToObject(), nil}, {"isdigit", wrapArgs("abc#123"), False.ToObject(), nil}, {"isdigit", wrapArgs("123", "456"), nil, mustCreateException(TypeErrorType, "'isdigit' of 'str' requires 1 arguments")}, {"islower", wrapArgs("abc"), True.ToObject(), nil}, {"islower", wrapArgs("ABC"), False.ToObject(), nil}, {"islower", wrapArgs(""), False.ToObject(), nil}, {"islower", wrapArgs("abc#123"), False.ToObject(), nil}, {"islower", wrapArgs("123", "456"), nil, mustCreateException(TypeErrorType, "'islower' of 'str' requires 1 arguments")}, {"isupper", wrapArgs("abc"), False.ToObject(), nil}, {"isupper", wrapArgs("ABC"), True.ToObject(), nil}, {"isupper", wrapArgs(""), False.ToObject(), nil}, {"isupper", wrapArgs("abc#123"), False.ToObject(), nil}, {"isupper", wrapArgs("123", "456"), nil, mustCreateException(TypeErrorType, "'isupper' of 'str' requires 1 arguments")}, {"isspace", wrapArgs(""), False.ToObject(), nil}, {"isspace", wrapArgs(" "), True.ToObject(), nil}, {"isspace", wrapArgs("\n\t\v\f\r "), True.ToObject(), nil}, {"isspace", wrapArgs(""), False.ToObject(), nil}, {"isspace", wrapArgs("asdad"), False.ToObject(), nil}, {"isspace", wrapArgs(" "), True.ToObject(), nil}, {"isspace", wrapArgs(" ", "456"), nil, mustCreateException(TypeErrorType, "'isspace' of 'str' requires 1 arguments")}, {"istitle", wrapArgs("abc"), False.ToObject(), nil}, {"istitle", wrapArgs("Abc&D"), True.ToObject(), nil}, {"istitle", wrapArgs("ABc&D"), False.ToObject(), nil}, {"istitle", wrapArgs(""), False.ToObject(), nil}, {"istitle", wrapArgs("abc#123"), False.ToObject(), nil}, {"istitle", wrapArgs("ABc&D", "456"), nil, mustCreateException(TypeErrorType, "'istitle' of 'str' requires 1 arguments")}, {"join", wrapArgs(",", newTestList("foo", "bar")), NewStr("foo,bar").ToObject(), nil}, {"join", wrapArgs(":", newTestList("foo", "bar", NewUnicode("baz"))), NewUnicode("foo:bar:baz").ToObject(), nil}, {"join", wrapArgs("nope", NewTuple()), NewStr("").ToObject(), nil}, {"join", wrapArgs("nope", newTestTuple("foo")), NewStr("foo").ToObject(), nil}, {"join", wrapArgs(",", newTestList("foo", "bar", 3.14)), nil, mustCreateException(TypeErrorType, "sequence item 2: expected string, float found")}, {"join", wrapArgs("\xff", newTestList(NewUnicode("foo"), NewUnicode("bar"))), nil, mustCreateException(UnicodeDecodeErrorType, "'utf8' codec can't decode byte 0xff in position 0")}, {"ljust", wrapArgs("foobar", 10, "#"), NewStr("foobar####").ToObject(), nil}, {"ljust", wrapArgs("foobar", 3, "#"), NewStr("foobar").ToObject(), nil}, {"ljust", wrapArgs("foobar", -1, "#"), NewStr("foobar").ToObject(), nil}, {"ljust", wrapArgs("foobar", 10, "##"), nil, mustCreateException(TypeErrorType, "ljust() argument 2 must be char, not str")}, {"ljust", wrapArgs("foobar", 10, ""), nil, mustCreateException(TypeErrorType, "ljust() argument 2 must be char, not str")}, {"lower", wrapArgs(""), NewStr("").ToObject(), nil}, {"lower", wrapArgs("a"), NewStr("a").ToObject(), nil}, {"lower", wrapArgs("A"), NewStr("a").ToObject(), nil}, {"lower", wrapArgs(" A"), NewStr(" a").ToObject(), nil}, {"lower", wrapArgs("abc"), NewStr("abc").ToObject(), nil}, {"lower", wrapArgs("ABC"), NewStr("abc").ToObject(), nil}, {"lower", wrapArgs("aBC"), NewStr("abc").ToObject(), nil}, {"lower", wrapArgs("abc def", 123), nil, mustCreateException(TypeErrorType, "'lower' of 'str' requires 1 arguments")}, {"lower", wrapArgs(123), nil, mustCreateException(TypeErrorType, "unbound method lower() must be called with str instance as first argument (got int instance instead)")}, {"lower", wrapArgs("вол"), NewStr("вол").ToObject(), nil}, {"lower", wrapArgs("ВОЛ"), NewStr("ВОЛ").ToObject(), nil}, {"lstrip", wrapArgs("foo "), NewStr("foo ").ToObject(), nil}, {"lstrip", wrapArgs(" foo bar "), NewStr("foo bar ").ToObject(), nil}, {"lstrip", wrapArgs("foo foo", "o"), NewStr("foo foo").ToObject(), nil}, {"lstrip", wrapArgs("foo foo", "f"), NewStr("oo foo").ToObject(), nil}, {"lstrip", wrapArgs("foo bar", "abr"), NewStr("foo bar").ToObject(), nil}, {"lstrip", wrapArgs("foo bar", "fo"), NewStr(" bar").ToObject(), nil}, {"lstrip", wrapArgs("foo", NewUnicode("f")), NewUnicode("oo").ToObject(), nil}, {"lstrip", wrapArgs("123", 3), nil, mustCreateException(TypeErrorType, "strip arg must be None, str or unicode")}, {"lstrip", wrapArgs("foo", "bar", "baz"), nil, mustCreateException(TypeErrorType, "'strip' of 'str' requires 2 arguments")}, {"lstrip", wrapArgs("\xfboo", NewUnicode("o")), nil, mustCreateException(UnicodeDecodeErrorType, "'utf8' codec can't decode byte 0xfb in position 0")}, {"lstrip", wrapArgs("foo", NewUnicode("o")), NewUnicode("f").ToObject(), nil}, {"rfind", wrapArgs("", ""), NewInt(0).ToObject(), nil}, {"rfind", wrapArgs("", "", 1), NewInt(-1).ToObject(), nil}, {"rfind", wrapArgs("", "", -1), NewInt(0).ToObject(), nil}, {"rfind", wrapArgs("", "", None, -1), NewInt(0).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar"), NewInt(3).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", fooType), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"rfind", wrapArgs("foobar", "bar", NewInt(MaxInt)), NewInt(-1).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", None, NewInt(MaxInt)), NewInt(3).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", newObject(intIndexType)), NewInt(3).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", None, newObject(intIndexType)), NewInt(-1).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", newObject(longIndexType)), NewInt(3).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", None, newObject(longIndexType)), NewInt(-1).ToObject(), nil}, //r TODO: Support unicode substring. {"rfind", wrapArgs("foobar", NewUnicode("bar")), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'unicode'")}, {"rfind", wrapArgs("foobar", "bar", "baz"), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"rfind", wrapArgs("foobar", "bar", 0, "baz"), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"rfind", wrapArgs("foobar", "bar", None), NewInt(3).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", 0, None), NewInt(3).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", 0, -2), NewInt(-1).ToObject(), nil}, {"rfind", wrapArgs("foobar", "foo", 0, 3), NewInt(0).ToObject(), nil}, {"rfind", wrapArgs("foobar", "foo", 10), NewInt(-1).ToObject(), nil}, {"rfind", wrapArgs("foobar", "foo", 3, 3), NewInt(-1).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", 3, 5), NewInt(-1).ToObject(), nil}, {"rfind", wrapArgs("foobar", "bar", 5, 3), NewInt(-1).ToObject(), nil}, {"rfind", wrapArgs("bar", "foobar"), NewInt(-1).ToObject(), nil}, {"rfind", wrapArgs("bar", "a", 1, 10), NewInt(1).ToObject(), nil}, {"rfind", wrapArgs("bar", "a", NewLong(big.NewInt(1)), 10), NewInt(1).ToObject(), nil}, {"rfind", wrapArgs("bar", "a", 0, NewLong(big.NewInt(2))), NewInt(1).ToObject(), nil}, {"rfind", wrapArgs("bar", "a", 1, 3), NewInt(1).ToObject(), nil}, {"rfind", wrapArgs("bar", "a", 0, -1), NewInt(1).ToObject(), nil}, {"rfind", wrapArgs("foo", newTestTuple("barfoo", "oo").ToObject()), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'tuple'")}, {"rfind", wrapArgs("foo", 123), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'int'")}, {"rfind", wrapArgs("barbaz", "ba"), NewInt(3).ToObject(), nil}, {"rfind", wrapArgs("barbaz", "ba", None, 4), NewInt(0).ToObject(), nil}, {"rindex", wrapArgs("", ""), NewInt(0).ToObject(), nil}, {"rindex", wrapArgs("", "", 1), nil, mustCreateException(ValueErrorType, "substring not found")}, {"rindex", wrapArgs("", "", -1), NewInt(0).ToObject(), nil}, {"rindex", wrapArgs("", "", None, -1), NewInt(0).ToObject(), nil}, {"rindex", wrapArgs("foobar", "bar"), NewInt(3).ToObject(), nil}, {"rindex", wrapArgs("foobar", "bar", fooType), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"rindex", wrapArgs("foobar", "bar", NewInt(MaxInt)), nil, mustCreateException(ValueErrorType, "substring not found")}, {"rindex", wrapArgs("foobar", "bar", None, NewInt(MaxInt)), NewInt(3).ToObject(), nil}, {"rindex", wrapArgs("foobar", "bar", newObject(intIndexType)), NewInt(3).ToObject(), nil}, {"rindex", wrapArgs("foobar", "bar", None, newObject(intIndexType)), nil, mustCreateException(ValueErrorType, "substring not found")}, {"rindex", wrapArgs("foobar", "bar", newObject(longIndexType)), NewInt(3).ToObject(), nil}, {"rindex", wrapArgs("foobar", "bar", None, newObject(longIndexType)), nil, mustCreateException(ValueErrorType, "substring not found")}, // TODO: Support unicode substring. {"rindex", wrapArgs("foobar", NewUnicode("bar")), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'unicode'")}, {"rindex", wrapArgs("foobar", "bar", "baz"), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"rindex", wrapArgs("foobar", "bar", 0, "baz"), nil, mustCreateException(TypeErrorType, "slice indices must be integers or None or have an __index__ method")}, {"rindex", wrapArgs("foobar", "bar", None), NewInt(3).ToObject(), nil}, {"rindex", wrapArgs("foobar", "bar", 0, None), NewInt(3).ToObject(), nil}, {"rindex", wrapArgs("foobar", "bar", 0, -2), nil, mustCreateException(ValueErrorType, "substring not found")}, {"rindex", wrapArgs("foobar", "foo", 0, 3), NewInt(0).ToObject(), nil}, {"rindex", wrapArgs("foobar", "foo", 10), nil, mustCreateException(ValueErrorType, "substring not found")}, {"rindex", wrapArgs("foobar", "foo", 3, 3), nil, mustCreateException(ValueErrorType, "substring not found")}, {"rindex", wrapArgs("foobar", "bar", 3, 5), nil, mustCreateException(ValueErrorType, "substring not found")}, {"rindex", wrapArgs("foobar", "bar", 5, 3), nil, mustCreateException(ValueErrorType, "substring not found")}, {"rindex", wrapArgs("bar", "foobar"), nil, mustCreateException(ValueErrorType, "substring not found")}, {"rindex", wrapArgs("bar", "a", 1, 10), NewInt(1).ToObject(), nil}, {"rindex", wrapArgs("bar", "a", NewLong(big.NewInt(1)), 10), NewInt(1).ToObject(), nil}, {"rindex", wrapArgs("bar", "a", 0, NewLong(big.NewInt(2))), NewInt(1).ToObject(), nil}, {"rindex", wrapArgs("bar", "a", 1, 3), NewInt(1).ToObject(), nil}, {"rindex", wrapArgs("bar", "a", 0, -1), NewInt(1).ToObject(), nil}, {"rindex", wrapArgs("foo", newTestTuple("barfoo", "oo").ToObject()), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'tuple'")}, {"rindex", wrapArgs("foo", 123), nil, mustCreateException(TypeErrorType, "'find/index' requires a 'str' object but received a 'int'")}, {"rindex", wrapArgs("barbaz", "ba"), NewInt(3).ToObject(), nil}, {"rindex", wrapArgs("barbaz", "ba", None, 4), NewInt(0).ToObject(), nil}, {"rjust", wrapArgs("foobar", 10, "#"), NewStr("####foobar").ToObject(), nil}, {"rjust", wrapArgs("foobar", 3, "#"), NewStr("foobar").ToObject(), nil}, {"rjust", wrapArgs("foobar", -1, "#"), NewStr("foobar").ToObject(), nil}, {"rjust", wrapArgs("foobar", 10, "##"), nil, mustCreateException(TypeErrorType, "rjust() argument 2 must be char, not str")}, {"rjust", wrapArgs("foobar", 10, ""), nil, mustCreateException(TypeErrorType, "rjust() argument 2 must be char, not str")}, {"split", wrapArgs("foo,bar", ","), newTestList("foo", "bar").ToObject(), nil}, {"split", wrapArgs("1,2,3", ",", 1), newTestList("1", "2,3").ToObject(), nil}, {"split", wrapArgs("a \tb\nc"), newTestList("a", "b", "c").ToObject(), nil}, {"split", wrapArgs("a \tb\nc", None), newTestList("a", "b", "c").ToObject(), nil}, {"split", wrapArgs("a \tb\nc", None, -1), newTestList("a", "b", "c").ToObject(), nil}, {"split", wrapArgs("a \tb\nc", None, 1), newTestList("a", "b\nc").ToObject(), nil}, {"split", wrapArgs("foo", 1), nil, mustCreateException(TypeErrorType, "expected a str separator")}, {"split", wrapArgs("foo", ""), nil, mustCreateException(ValueErrorType, "empty separator")}, {"split", wrapArgs(""), newTestList().ToObject(), nil}, {"split", wrapArgs(" "), newTestList().ToObject(), nil}, {"split", wrapArgs("", "x"), newTestList("").ToObject(), nil}, {"split", wrapArgs(" ", " ", 1), newTestList("", "").ToObject(), nil}, {"split", wrapArgs("aa", "a", 2), newTestList("", "", "").ToObject(), nil}, {"split", wrapArgs(" a ", "a"), newTestList(" ", " ").ToObject(), nil}, {"split", wrapArgs("a b c d", None, 1), newTestList("a", "b c d").ToObject(), nil}, {"split", wrapArgs("a b c d "), newTestList("a", "b", "c", "d").ToObject(), nil}, {"split", wrapArgs(" a b c d ", None, 1), newTestList("a", "b c d ").ToObject(), nil}, {"split", wrapArgs(" a b c d ", None, 0), newTestList("a b c d ").ToObject(), nil}, {"splitlines", wrapArgs(""), NewList().ToObject(), nil}, {"splitlines", wrapArgs("\n"), newTestList("").ToObject(), nil}, {"splitlines", wrapArgs("foo"), newTestList("foo").ToObject(), nil}, {"splitlines", wrapArgs("foo\r"), newTestList("foo").ToObject(), nil}, {"splitlines", wrapArgs("foo\r", true), newTestList("foo\r").ToObject(), nil}, {"splitlines", wrapArgs("foo\r\nbar\n", big.NewInt(12)), newTestList("foo\r\n", "bar\n").ToObject(), nil}, {"splitlines", wrapArgs("foo\n\r\nbar\n\n"), newTestList("foo", "", "bar", "").ToObject(), nil}, {"splitlines", wrapArgs("foo", newObject(ObjectType)), nil, mustCreateException(TypeErrorType, "an integer is required")}, {"splitlines", wrapArgs("foo", "bar", "baz"), nil, mustCreateException(TypeErrorType, "'splitlines' of 'str' requires 2 arguments")}, {"splitlines", wrapArgs("foo", overflowLong), nil, mustCreateException(OverflowErrorType, "Python int too large to convert to a Go int")}, {"startswith", wrapArgs("", ""), True.ToObject(), nil}, {"startswith", wrapArgs("", "", 1), False.ToObject(), nil}, {"startswith", wrapArgs("foobar", "foo"), True.ToObject(), nil}, {"startswith", wrapArgs("foobar", "foo", 2), False.ToObject(), nil}, {"startswith", wrapArgs("foobar", "bar", 3), True.ToObject(), nil}, {"startswith", wrapArgs("foobar", "bar", 3, 5), False.ToObject(), nil}, {"startswith", wrapArgs("foobar", "bar", 5, 3), False.ToObject(), nil}, {"startswith", wrapArgs("foo", "foobar"), False.ToObject(), nil}, {"startswith", wrapArgs("foo", newTestTuple("foobar", "fo").ToObject()), True.ToObject(), nil}, {"startswith", wrapArgs("foo", 123), nil, mustCreateException(TypeErrorType, "startswith first arg must be str, unicode, or tuple, not int")}, {"startswith", wrapArgs("foo", "f", "123"), nil, mustCreateException(TypeErrorType, "'startswith' requires a 'int' object but received a 'str'")}, {"startswith", wrapArgs("foo", newTestTuple(123).ToObject()), nil, mustCreateException(TypeErrorType, "expected a str")}, {"strip", wrapArgs("foo "), NewStr("foo").ToObject(), nil}, {"strip", wrapArgs(" foo bar "), NewStr("foo bar").ToObject(), nil}, {"strip", wrapArgs("foo foo", "o"), NewStr("foo f").ToObject(), nil}, {"strip", wrapArgs("foo bar", "abr"), NewStr("foo ").ToObject(), nil}, {"strip", wrapArgs("foo", NewUnicode("o")), NewUnicode("f").ToObject(), nil}, {"strip", wrapArgs("123", 3), nil, mustCreateException(TypeErrorType, "strip arg must be None, str or unicode")}, {"strip", wrapArgs("foo", "bar", "baz"), nil, mustCreateException(TypeErrorType, "'strip' of 'str' requires 2 arguments")}, {"strip", wrapArgs("\xfboo", NewUnicode("o")), nil, mustCreateException(UnicodeDecodeErrorType, "'utf8' codec can't decode byte 0xfb in position 0")}, {"strip", wrapArgs("foo", NewUnicode("o")), NewUnicode("f").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "!", "@", 1), NewStr("one@two!three!").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "!", ""), NewStr("onetwothree").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "!", "@", 2), NewStr("one@two@three!").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "!", "@", 3), NewStr("one@two@three@").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "!", "@", 4), NewStr("one@two@three@").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "!", "@", 0), NewStr("one!two!three!").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "!", "@"), NewStr("one@two@three@").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "x", "@"), NewStr("one!two!three!").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "x", "@", 2), NewStr("one!two!three!").ToObject(), nil}, {"replace", wrapArgs("\xd0\xb2\xd0\xbe\xd0\xbb", "", "\x00", -1), NewStr("\x00\xd0\x00\xb2\x00\xd0\x00\xbe\x00\xd0\x00\xbb\x00").ToObject(), nil}, {"replace", wrapArgs("\xd0\xb2\xd0\xbe\xd0\xbb", "", "\x01\x02", -1), NewStr("\x01\x02\xd0\x01\x02\xb2\x01\x02\xd0\x01\x02\xbe\x01\x02\xd0\x01\x02\xbb\x01\x02").ToObject(), nil}, {"replace", wrapArgs("abc", "", "-"), NewStr("-a-b-c-").ToObject(), nil}, {"replace", wrapArgs("abc", "", "-", 3), NewStr("-a-b-c").ToObject(), nil}, {"replace", wrapArgs("abc", "", "-", 0), NewStr("abc").ToObject(), nil}, {"replace", wrapArgs("", "", ""), NewStr("").ToObject(), nil}, {"replace", wrapArgs("", "", "a"), NewStr("a").ToObject(), nil}, {"replace", wrapArgs("abc", "a", "--", 0), NewStr("abc").ToObject(), nil}, {"replace", wrapArgs("abc", "xy", "--"), NewStr("abc").ToObject(), nil}, {"replace", wrapArgs("123", "123", ""), NewStr("").ToObject(), nil}, {"replace", wrapArgs("123123", "123", ""), NewStr("").ToObject(), nil}, {"replace", wrapArgs("123x123", "123", ""), NewStr("x").ToObject(), nil}, {"replace", wrapArgs("one!two!three!", "!", "@", NewLong(big.NewInt(1))), NewStr("one@two!three!").ToObject(), nil}, {"replace", wrapArgs("foobar", "bar", "baz", newObject(intIntType)), NewStr("foobaz").ToObject(), nil}, {"replace", wrapArgs("foobar", "bar", "baz", newObject(longIntType)), NewStr("foobaz").ToObject(), nil}, {"replace", wrapArgs("", "", "x"), NewStr("x").ToObject(), nil}, {"replace", wrapArgs("", "", "x", -1), NewStr("x").ToObject(), nil}, {"replace", wrapArgs("", "", "x", 0), NewStr("").ToObject(), nil}, {"replace", wrapArgs("", "", "x", 1), NewStr("").ToObject(), nil}, {"replace", wrapArgs("", "", "x", 1000), NewStr("").ToObject(), nil}, // TODO: Support unicode substring. {"replace", wrapArgs("foobar", "", NewUnicode("bar")), nil, mustCreateException(TypeErrorType, "'replace' requires a 'str' object but received a 'unicode'")}, {"replace", wrapArgs("foobar", NewUnicode("bar"), ""), nil, mustCreateException(TypeErrorType, "'replace' requires a 'str' object but received a 'unicode'")}, {"replace", wrapArgs("foobar", "bar", "baz", None), nil, mustCreateException(TypeErrorType, "an integer is required")}, {"replace", wrapArgs("foobar", "bar", "baz", newObject(intIndexType)), nil, mustCreateException(TypeErrorType, "an integer is required")}, {"replace", wrapArgs("foobar", "bar", "baz", newObject(longIndexType)), nil, mustCreateException(TypeErrorType, "an integer is required")}, {"rstrip", wrapArgs("foo "), NewStr("foo").ToObject(), nil}, {"rstrip", wrapArgs(" foo bar "), NewStr(" foo bar").ToObject(), nil}, {"rstrip", wrapArgs("foo foo", "o"), NewStr("foo f").ToObject(), nil}, {"rstrip", wrapArgs("foo bar", "abr"), NewStr("foo ").ToObject(), nil}, {"rstrip", wrapArgs("foo", NewUnicode("o")), NewUnicode("f").ToObject(), nil}, {"rstrip", wrapArgs("123", 3), nil, mustCreateException(TypeErrorType, "strip arg must be None, str or unicode")}, {"rstrip", wrapArgs("foo", "bar", "baz"), nil, mustCreateException(TypeErrorType, "'strip' of 'str' requires 2 arguments")}, {"rstrip", wrapArgs("\xfboo", NewUnicode("o")), nil, mustCreateException(UnicodeDecodeErrorType, "'utf8' codec can't decode byte 0xfb in position 0")}, {"rstrip", wrapArgs("foo", NewUnicode("o")), NewUnicode("f").ToObject(), nil}, {"title", wrapArgs(""), NewStr("").ToObject(), nil}, {"title", wrapArgs("a"), NewStr("A").ToObject(), nil}, {"title", wrapArgs("A"), NewStr("A").ToObject(), nil}, {"title", wrapArgs(" a"), NewStr(" A").ToObject(), nil}, {"title", wrapArgs("abc def"), NewStr("Abc Def").ToObject(), nil}, {"title", wrapArgs("ABC DEF"), NewStr("Abc Def").ToObject(), nil}, {"title", wrapArgs("aBC dEF"), NewStr("Abc Def").ToObject(), nil}, {"title", wrapArgs("abc def", 123), nil, mustCreateException(TypeErrorType, "'title' of 'str' requires 1 arguments")}, {"title", wrapArgs(123), nil, mustCreateException(TypeErrorType, "unbound method title() must be called with str instance as first argument (got int instance instead)")}, {"title", wrapArgs("вол"), NewStr("вол").ToObject(), nil}, {"title", wrapArgs("ВОЛ"), NewStr("ВОЛ").ToObject(), nil}, {"upper", wrapArgs(""), NewStr("").ToObject(), nil}, {"upper", wrapArgs("a"), NewStr("A").ToObject(), nil}, {"upper", wrapArgs("A"), NewStr("A").ToObject(), nil}, {"upper", wrapArgs(" a"), NewStr(" A").ToObject(), nil}, {"upper", wrapArgs("abc"), NewStr("ABC").ToObject(), nil}, {"upper", wrapArgs("ABC"), NewStr("ABC").ToObject(), nil}, {"upper", wrapArgs("aBC"), NewStr("ABC").ToObject(), nil}, {"upper", wrapArgs("abc def", 123), nil, mustCreateException(TypeErrorType, "'upper' of 'str' requires 1 arguments")}, {"upper", wrapArgs(123), nil, mustCreateException(TypeErrorType, "unbound method upper() must be called with str instance as first argument (got int instance instead)")}, {"upper", wrapArgs("вол"), NewStr("вол").ToObject(), nil}, {"upper", wrapArgs("ВОЛ"), NewStr("ВОЛ").ToObject(), nil}, {"zfill", wrapArgs("123", 2), NewStr("123").ToObject(), nil}, {"zfill", wrapArgs("123", 3), NewStr("123").ToObject(), nil}, {"zfill", wrapArgs("123", 4), NewStr("0123").ToObject(), nil}, {"zfill", wrapArgs("+123", 3), NewStr("+123").ToObject(), nil}, {"zfill", wrapArgs("+123", 4), NewStr("+123").ToObject(), nil}, {"zfill", wrapArgs("+123", 5), NewStr("+0123").ToObject(), nil}, {"zfill", wrapArgs("-123", 3), NewStr("-123").ToObject(), nil}, {"zfill", wrapArgs("-123", 4), NewStr("-123").ToObject(), nil}, {"zfill", wrapArgs("-123", 5), NewStr("-0123").ToObject(), nil}, {"zfill", wrapArgs("123", NewLong(big.NewInt(3))), NewStr("123").ToObject(), nil}, {"zfill", wrapArgs("123", NewLong(big.NewInt(5))), NewStr("00123").ToObject(), nil}, {"zfill", wrapArgs("", 0), NewStr("").ToObject(), nil}, {"zfill", wrapArgs("", 1), NewStr("0").ToObject(), nil}, {"zfill", wrapArgs("", 3), NewStr("000").ToObject(), nil}, {"zfill", wrapArgs("", -1), NewStr("").ToObject(), nil}, {"zfill", wrapArgs("34", 1), NewStr("34").ToObject(), nil}, {"zfill", wrapArgs("34", 4), NewStr("0034").ToObject(), nil}, {"zfill", wrapArgs("34", None), nil, mustCreateException(TypeErrorType, "an integer is required")}, {"zfill", wrapArgs("", True), NewStr("0").ToObject(), nil}, {"zfill", wrapArgs("", False), NewStr("").ToObject(), nil}, {"zfill", wrapArgs("34", NewStr("test")), nil, mustCreateException(TypeErrorType, "an integer is required")}, {"zfill", wrapArgs("34"), nil, mustCreateException(TypeErrorType, "'zfill' of 'str' requires 2 arguments")}, {"swapcase", wrapArgs(""), NewStr("").ToObject(), nil}, {"swapcase", wrapArgs("a"), NewStr("A").ToObject(), nil}, {"swapcase", wrapArgs("A"), NewStr("a").ToObject(), nil}, {"swapcase", wrapArgs(" A"), NewStr(" a").ToObject(), nil}, {"swapcase", wrapArgs("abc"), NewStr("ABC").ToObject(), nil}, {"swapcase", wrapArgs("ABC"), NewStr("abc").ToObject(), nil}, {"swapcase", wrapArgs("aBC"), NewStr("Abc").ToObject(), nil}, {"swapcase", wrapArgs("abc def", 123), nil, mustCreateException(TypeErrorType, "'swapcase' of 'str' requires 1 arguments")}, {"swapcase", wrapArgs(123), nil, mustCreateException(TypeErrorType, "unbound method swapcase() must be called with str instance as first argument (got int instance instead)")}, {"swapcase", wrapArgs("вол"), NewStr("вол").ToObject(), nil}, {"swapcase", wrapArgs("ВОЛ"), NewStr("ВОЛ").ToObject(), nil}, } for _, cas := range cases { testCase := invokeTestCase{args: cas.args, want: cas.want, wantExc: cas.wantExc} if err := runInvokeMethodTestCase(StrType, cas.methodName, &testCase); err != "" { t.Error(err) } } } func TestStrStr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs("foo"), want: NewStr("foo").ToObject()}, {args: wrapArgs("on\nmultiple\nlines"), want: NewStr("on\nmultiple\nlines").ToObject()}, {args: wrapArgs("\x00\x00"), want: NewStr("\x00\x00").ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(StrType, "__str__", &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/super.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "reflect" ) var ( // superType is the object representing the Python 'super' type. superType = newBasisType("super", reflect.TypeOf(super{}), toSuperUnsafe, ObjectType) ) type super struct { Object sub *Type obj *Object objType *Type } func toSuperUnsafe(o *Object) *super { return (*super)(o.toPointer()) } func superInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) { // TODO: Support the unbound form of super. if raised := checkFunctionArgs(f, "__init__", args, TypeType, ObjectType); raised != nil { return nil, raised } sup := toSuperUnsafe(o) sub := toTypeUnsafe(args[0]) obj := args[1] var objType *Type if obj.isInstance(TypeType) && toTypeUnsafe(obj).isSubclass(sub) { objType = toTypeUnsafe(obj) } else if obj.isInstance(sub) { objType = obj.typ } else { return nil, f.RaiseType(TypeErrorType, "super(type, obj): obj must be an instance or subtype of type") } sup.sub = sub sup.obj = obj sup.objType = objType return None, nil } func superGetAttribute(f *Frame, o *Object, name *Str) (*Object, *BaseException) { sup := toSuperUnsafe(o) // Tell the truth about the __class__ attribute. if sup.objType != nil && name.Value() != "__class__" { mro := sup.objType.mro n := len(mro) // Start from the immediate mro successor to the specified type. i := 0 for i < n && mro[i] != sup.sub { i++ } i++ var inst *Object if sup.obj != sup.objType.ToObject() { inst = sup.obj } // Now do normal mro lookup from the successor type. for ; i < n; i++ { dict := mro[i].Dict() res, raised := dict.GetItem(f, name.ToObject()) if raised != nil { return nil, raised } if res != nil { if get := res.typ.slots.Get; get != nil { // Found a descriptor so invoke it. return get.Fn(f, res, inst, sup.objType) } return res, nil } } } // Attribute not found on base classes so lookup the attr on the super // object itself. Most likely will AttributeError. return objectGetAttribute(f, o, name) } func initSuperType(map[string]*Object) { superType.slots.GetAttribute = &getAttributeSlot{superGetAttribute} superType.slots.Init = &initSlot{superInit} } ================================================ FILE: runtime/super_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestSuperInitErrors(t *testing.T) { // Only tests __init__ error cases. Non-error cases are tested // implicitly by TestSuperGetAttribute. cases := []invokeTestCase{ {wantExc: mustCreateException(TypeErrorType, "'__init__' requires 2 arguments")}, {args: wrapArgs(FloatType, 123), wantExc: mustCreateException(TypeErrorType, "super(type, obj): obj must be an instance or subtype of type")}, } for _, cas := range cases { if err := runInvokeTestCase(superType.ToObject(), &cas); err != "" { t.Error(err) } } } func TestSuperGetAttribute(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, t *Type, o *Object) (*Object, *BaseException) { sup, raised := superType.Call(f, wrapArgs(t, o), nil) if raised != nil { return nil, raised } return GetAttr(f, sup, NewStr("attr"), nil) }) // top, left, bottom, right refer to parts of a diamond hierarchy. topType := newTestClass("Top", []*Type{ObjectType}, newStringDict(map[string]*Object{ "attr": NewStr("top").ToObject(), })) top := newObject(topType) leftType := newTestClass("Left", []*Type{topType}, newStringDict(map[string]*Object{ "attr": NewStr("left").ToObject(), })) left := newObject(leftType) rightType := newTestClass("Right", []*Type{topType}, newStringDict(map[string]*Object{ "attr": newProperty(newBuiltinFunction("attr", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { s := "right" if args[0] == nil { // When the "instance" argument is nil, the // descriptor is unbound. s = "rightType" } return NewStr(s).ToObject(), nil }).ToObject(), nil, nil).ToObject(), })) right := newObject(rightType) bottomType := newTestClass("Bottom", []*Type{leftType, rightType}, newStringDict(map[string]*Object{ "attr": NewStr("bottom").ToObject(), })) bottom := newObject(bottomType) cases := []invokeTestCase{ {args: wrapArgs(bottomType, bottom), want: NewStr("left").ToObject()}, {args: wrapArgs(bottomType, bottomType), want: NewStr("left").ToObject()}, {args: wrapArgs(leftType, bottom), want: NewStr("right").ToObject()}, {args: wrapArgs(leftType, bottomType), want: NewStr("rightType").ToObject()}, {args: wrapArgs(rightType, bottom), want: NewStr("top").ToObject()}, {args: wrapArgs(rightType, bottomType), want: NewStr("top").ToObject()}, {args: wrapArgs(topType, bottom), wantExc: mustCreateException(AttributeErrorType, "'super' object has no attribute 'attr'")}, {args: wrapArgs(topType, bottomType), wantExc: mustCreateException(AttributeErrorType, "'super' object has no attribute 'attr'")}, {args: wrapArgs(leftType, left), want: NewStr("top").ToObject()}, {args: wrapArgs(leftType, leftType), want: NewStr("top").ToObject()}, {args: wrapArgs(topType, left), wantExc: mustCreateException(AttributeErrorType, "'super' object has no attribute 'attr'")}, {args: wrapArgs(topType, leftType), wantExc: mustCreateException(AttributeErrorType, "'super' object has no attribute 'attr'")}, {args: wrapArgs(rightType, right), want: NewStr("top").ToObject()}, {args: wrapArgs(rightType, rightType), want: NewStr("top").ToObject()}, {args: wrapArgs(topType, right), wantExc: mustCreateException(AttributeErrorType, "'super' object has no attribute 'attr'")}, {args: wrapArgs(topType, rightType), wantExc: mustCreateException(AttributeErrorType, "'super' object has no attribute 'attr'")}, {args: wrapArgs(topType, top), wantExc: mustCreateException(AttributeErrorType, "'super' object has no attribute 'attr'")}, {args: wrapArgs(topType, topType), wantExc: mustCreateException(AttributeErrorType, "'super' object has no attribute 'attr'")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/threading.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "sync" "sync/atomic" "unsafe" ) const ( argsCacheSize = 16 argsCacheArgc = 6 ) type threadState struct { reprState map[*Object]bool excValue *BaseException excTraceback *Traceback // argsCache is a small, per-thread LIFO cache for arg lists. Entries // have a fixed capacity so calls to functions with larger parameter // lists will be allocated afresh each time. Args freed when the cache // is full are dropped. If the cache is empty then a new args slice // will be allocated. argsCache []Args // frameCache is a local cache of allocated frames almost ready for // reuse. The cache is maintained through the Frame `back` pointer as a // singly linked list. frameCache *Frame } func newThreadState() *threadState { return &threadState{argsCache: make([]Args, 0, argsCacheSize)} } // recursiveMutex implements a typical reentrant lock, similar to Python's // RLock. Lock can be called multiple times for the same frame stack. type recursiveMutex struct { mutex sync.Mutex threadState *threadState count int } func (m *recursiveMutex) Lock(f *Frame) { p := (*unsafe.Pointer)(unsafe.Pointer(&m.threadState)) if (*threadState)(atomic.LoadPointer(p)) != f.threadState { // m.threadState != f.threadState implies m is not held by this // thread and therefore we won't deadlock acquiring the mutex. m.mutex.Lock() // m.threadState is now guaranteed to be empty (otherwise we // couldn't have acquired m.mutex) so store our own thread ID. atomic.StorePointer(p, unsafe.Pointer(f.threadState)) m.count++ } else { m.count++ } } func (m *recursiveMutex) Unlock(f *Frame) { p := (*unsafe.Pointer)(unsafe.Pointer(&m.threadState)) if (*threadState)(atomic.LoadPointer(p)) != f.threadState { logFatal("recursiveMutex.Unlock: frame did not match that passed to Lock") } // Since we're unlocking, we must hold m.mutex, so this is safe. if m.count <= 0 { logFatal("recursiveMutex.Unlock: Unlock called too many times") } m.count-- if m.count == 0 { atomic.StorePointer(p, unsafe.Pointer(nil)) m.mutex.Unlock() } } // TryableMutex is a mutex-like object that also supports TryLock(). type TryableMutex struct { c chan bool } // NewTryableMutex returns a new TryableMutex. func NewTryableMutex() *TryableMutex { m := &TryableMutex{make(chan bool, 1)} m.Unlock() return m } // Lock blocks until the mutex is available and then acquires a lock. func (m *TryableMutex) Lock() { <-m.c } // TryLock returns true and acquires a lock if the mutex is available, otherwise // it returns false. func (m *TryableMutex) TryLock() bool { select { case <-m.c: return true default: return false } } // Unlock releases the mutex's lock. func (m *TryableMutex) Unlock() { m.c <- true } ================================================ FILE: runtime/threading_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "testing" ) func TestRecursiveMutex(t *testing.T) { var m recursiveMutex f := NewRootFrame() m.Lock(f) m.Lock(f) m.Unlock(f) m.Unlock(f) } func TestRecursiveMutexUnlockedTooManyTimes(t *testing.T) { var m recursiveMutex f := NewRootFrame() m.Lock(f) m.Unlock(f) oldLogFatal := logFatal logFatal = func(msg string) { panic(msg) } defer func() { logFatal = oldLogFatal if e := recover(); e == nil { t.Error("Unlock didn't call logFatal") } }() m.Unlock(f) } func TestRecursiveMutexUnlockFrameMismatch(t *testing.T) { var m recursiveMutex m.Lock(NewRootFrame()) oldLogFatal := logFatal logFatal = func(msg string) { panic(msg) } defer func() { logFatal = oldLogFatal if e := recover(); e == nil { t.Error("Unlock didn't call logFatal") } }() m.Unlock(NewRootFrame()) } ================================================ FILE: runtime/traceback.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "reflect" ) // Traceback represents Python 'traceback' objects. type Traceback struct { Object frame *Frame `attr:"tb_frame"` next *Traceback `attr:"tb_next"` lineno int `attr:"tb_lineno"` } func newTraceback(f *Frame, next *Traceback) *Traceback { f.taken = true return &Traceback{Object{typ: TracebackType}, f, next, f.lineno} } func toTracebackUnsafe(o *Object) *Traceback { return (*Traceback)(o.toPointer()) } // ToObject upcasts f to an Object. func (f *Traceback) ToObject() *Object { return &f.Object } // TracebackType is the object representing the Python 'traceback' type. var TracebackType = newBasisType("traceback", reflect.TypeOf(Traceback{}), toTracebackUnsafe, ObjectType) func initTracebackType(map[string]*Object) { TracebackType.flags &^= typeFlagInstantiable | typeFlagBasetype } ================================================ FILE: runtime/tuple.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" ) // Tuple represents Python 'tuple' objects. // // Tuples are thread safe by virtue of being immutable. type Tuple struct { Object elems []*Object } // NewTuple returns a tuple containing the given elements. func NewTuple(elems ...*Object) *Tuple { if len(elems) == 0 { return emptyTuple } return &Tuple{Object: Object{typ: TupleType}, elems: elems} } // Below are direct allocation versions of small Tuples. Rather than performing // two allocations, one for the tuple object and one for the slice holding the // elements, we allocate both objects at the same time in one block of memory. // This both decreases the number of allocations overall as well as increases // memory locality for tuple data. Both of which *should* improve time to // allocate as well as read performance. The methods below are used by the // compiler to create fixed size tuples when the size is known ahead of time. // // The number of specializations below were chosen first to cover all the fixed // size tuple allocations in the runtime (currently 5), then filled out to // cover the whole memory size class (see golang/src/runtime/sizeclasses.go for // the table). On a 64bit system, a tuple of length 6 occupies 96 bytes - 48 // bytes for the tuple object and 6*8 (48) bytes of pointers. // // If methods are added or removed, then the constant MAX_DIRECT_TUPLE in // compiler/util.py needs to be updated as well. // NewTuple0 returns the empty tuple. This is mostly provided for the // convenience of the compiler. func NewTuple0() *Tuple { return emptyTuple } // NewTuple1 returns a tuple of length 1 containing just elem0. func NewTuple1(elem0 *Object) *Tuple { t := struct { tuple Tuple elems [1]*Object }{ tuple: Tuple{Object: Object{typ: TupleType}}, elems: [1]*Object{elem0}, } t.tuple.elems = t.elems[:] return &t.tuple } // NewTuple2 returns a tuple of length 2 containing just elem0 and elem1. func NewTuple2(elem0, elem1 *Object) *Tuple { t := struct { tuple Tuple elems [2]*Object }{ tuple: Tuple{Object: Object{typ: TupleType}}, elems: [2]*Object{elem0, elem1}, } t.tuple.elems = t.elems[:] return &t.tuple } // NewTuple3 returns a tuple of length 3 containing elem0 to elem2. func NewTuple3(elem0, elem1, elem2 *Object) *Tuple { t := struct { tuple Tuple elems [3]*Object }{ tuple: Tuple{Object: Object{typ: TupleType}}, elems: [3]*Object{elem0, elem1, elem2}, } t.tuple.elems = t.elems[:] return &t.tuple } // NewTuple4 returns a tuple of length 4 containing elem0 to elem3. func NewTuple4(elem0, elem1, elem2, elem3 *Object) *Tuple { t := struct { tuple Tuple elems [4]*Object }{ tuple: Tuple{Object: Object{typ: TupleType}}, elems: [4]*Object{elem0, elem1, elem2, elem3}, } t.tuple.elems = t.elems[:] return &t.tuple } // NewTuple5 returns a tuple of length 5 containing elem0 to elem4. func NewTuple5(elem0, elem1, elem2, elem3, elem4 *Object) *Tuple { t := struct { tuple Tuple elems [5]*Object }{ tuple: Tuple{Object: Object{typ: TupleType}}, elems: [5]*Object{elem0, elem1, elem2, elem3, elem4}, } t.tuple.elems = t.elems[:] return &t.tuple } // NewTuple6 returns a tuple of length 6 containing elem0 to elem5. func NewTuple6(elem0, elem1, elem2, elem3, elem4, elem5 *Object) *Tuple { t := struct { tuple Tuple elems [6]*Object }{ tuple: Tuple{Object: Object{typ: TupleType}}, elems: [6]*Object{elem0, elem1, elem2, elem3, elem4, elem5}, } t.tuple.elems = t.elems[:] return &t.tuple } func toTupleUnsafe(o *Object) *Tuple { return (*Tuple)(o.toPointer()) } // GetItem returns the i'th element of t. Bounds are unchecked and therefore // this method will panic unless 0 <= i < t.Len(). func (t *Tuple) GetItem(i int) *Object { return t.elems[i] } // Len returns the number of elements in t. func (t *Tuple) Len() int { return len(t.elems) } // ToObject upcasts t to an Object. func (t *Tuple) ToObject() *Object { return &t.Object } // TupleType is the object representing the Python 'tuple' type. var TupleType = newBasisType("tuple", reflect.TypeOf(Tuple{}), toTupleUnsafe, ObjectType) var emptyTuple = &Tuple{Object: Object{typ: TupleType}} func tupleAdd(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(TupleType) { return NotImplemented, nil } elems, raised := seqAdd(f, toTupleUnsafe(v).elems, toTupleUnsafe(w).elems) if raised != nil { return nil, raised } return NewTuple(elems...).ToObject(), nil } func tupleContains(f *Frame, t, v *Object) (*Object, *BaseException) { return seqContains(f, t, v) } func tupleCount(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "count", args, TupleType, ObjectType); raised != nil { return nil, raised } return seqCount(f, args[0], args[1]) } func tupleEq(f *Frame, v, w *Object) (*Object, *BaseException) { return tupleCompare(f, toTupleUnsafe(v), w, Eq) } func tupleGE(f *Frame, v, w *Object) (*Object, *BaseException) { return tupleCompare(f, toTupleUnsafe(v), w, GE) } func tupleGetItem(f *Frame, o, key *Object) (*Object, *BaseException) { t := toTupleUnsafe(o) item, elems, raised := seqGetItem(f, t.elems, key) if raised != nil { return nil, raised } if item != nil { return item, nil } return NewTuple(elems...).ToObject(), nil } func tupleGetNewArgs(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__getnewargs__", args, TupleType); raised != nil { return nil, raised } return NewTuple1(args[0]).ToObject(), nil } func tupleGT(f *Frame, v, w *Object) (*Object, *BaseException) { return tupleCompare(f, toTupleUnsafe(v), w, GT) } func tupleIter(f *Frame, o *Object) (*Object, *BaseException) { return newSliceIterator(reflect.ValueOf(toTupleUnsafe(o).elems)), nil } func tupleLE(f *Frame, v, w *Object) (*Object, *BaseException) { return tupleCompare(f, toTupleUnsafe(v), w, LE) } func tupleLen(f *Frame, o *Object) (*Object, *BaseException) { return NewInt(len(toTupleUnsafe(o).elems)).ToObject(), nil } func tupleLT(f *Frame, v, w *Object) (*Object, *BaseException) { return tupleCompare(f, toTupleUnsafe(v), w, LT) } func tupleMul(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } elems, raised := seqMul(f, toTupleUnsafe(v).elems, toIntUnsafe(w).Value()) if raised != nil { return nil, raised } return NewTuple(elems...).ToObject(), nil } func tupleNE(f *Frame, v, w *Object) (*Object, *BaseException) { return tupleCompare(f, toTupleUnsafe(v), w, NE) } func tupleNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { if t == TupleType && len(args) == 1 && args[0].typ == TupleType { // Tuples are immutable so just return the tuple provided. return args[0], nil } elems, raised := seqNew(f, args) if raised != nil { return nil, raised } tup := toTupleUnsafe(newObject(t)) tup.elems = elems return tup.ToObject(), nil } func tupleRepr(f *Frame, o *Object) (*Object, *BaseException) { t := toTupleUnsafe(o) if f.reprEnter(t.ToObject()) { return NewStr("(...)").ToObject(), nil } s, raised := seqRepr(f, t.elems) f.reprLeave(t.ToObject()) if raised != nil { return nil, raised } if len(t.elems) == 1 { s = fmt.Sprintf("(%s,)", s) } else { s = fmt.Sprintf("(%s)", s) } return NewStr(s).ToObject(), nil } func tupleRMul(f *Frame, v, w *Object) (*Object, *BaseException) { if !w.isInstance(IntType) { return NotImplemented, nil } elems, raised := seqMul(f, toTupleUnsafe(v).elems, toIntUnsafe(w).Value()) if raised != nil { return nil, raised } return NewTuple(elems...).ToObject(), nil } func initTupleType(dict map[string]*Object) { dict["count"] = newBuiltinFunction("count", tupleCount).ToObject() dict["__getnewargs__"] = newBuiltinFunction("__getnewargs__", tupleGetNewArgs).ToObject() TupleType.slots.Add = &binaryOpSlot{tupleAdd} TupleType.slots.Contains = &binaryOpSlot{tupleContains} TupleType.slots.Eq = &binaryOpSlot{tupleEq} TupleType.slots.GE = &binaryOpSlot{tupleGE} TupleType.slots.GetItem = &binaryOpSlot{tupleGetItem} TupleType.slots.GT = &binaryOpSlot{tupleGT} TupleType.slots.Iter = &unaryOpSlot{tupleIter} TupleType.slots.LE = &binaryOpSlot{tupleLE} TupleType.slots.Len = &unaryOpSlot{tupleLen} TupleType.slots.LT = &binaryOpSlot{tupleLT} TupleType.slots.Mul = &binaryOpSlot{tupleMul} TupleType.slots.NE = &binaryOpSlot{tupleNE} TupleType.slots.New = &newSlot{tupleNew} TupleType.slots.Repr = &unaryOpSlot{tupleRepr} TupleType.slots.RMul = &binaryOpSlot{tupleRMul} } func tupleCompare(f *Frame, v *Tuple, w *Object, cmp binaryOpFunc) (*Object, *BaseException) { if !w.isInstance(TupleType) { return NotImplemented, nil } return seqCompare(f, v.elems, toTupleUnsafe(w).elems, cmp) } ================================================ FILE: runtime/tuple_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "reflect" "testing" ) func TestNewTuple(t *testing.T) { cases := [][]*Object{ nil, {newObject(ObjectType)}, {newObject(ObjectType), newObject(ObjectType)}, } for _, args := range cases { tuple := NewTuple(args...) if !reflect.DeepEqual(tuple.elems, args) { t.Errorf("NewTuple(%v) = %v, want %v", args, tuple.elems, args) } } } func TestTupleBinaryOps(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, fn binaryOpFunc, v, w *Object) (*Object, *BaseException) { return fn(f, v, w) }) cases := []invokeTestCase{ {args: wrapArgs(Add, newTestTuple(3), newTestTuple("foo")), want: newTestTuple(3, "foo").ToObject()}, {args: wrapArgs(Add, NewTuple(None), NewTuple()), want: NewTuple(None).ToObject()}, {args: wrapArgs(Add, NewTuple(), newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'tuple' and 'object'")}, {args: wrapArgs(Add, None, NewTuple()), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'NoneType' and 'tuple'")}, {args: wrapArgs(Mul, NewTuple(), 10), want: NewTuple().ToObject()}, {args: wrapArgs(Mul, newTestTuple("baz"), -2), want: NewTuple().ToObject()}, {args: wrapArgs(Mul, newTestTuple(None, None), 0), want: NewTuple().ToObject()}, {args: wrapArgs(Mul, newTestTuple(1, "bar"), 2), want: newTestTuple(1, "bar", 1, "bar").ToObject()}, {args: wrapArgs(Mul, 1, newTestTuple(1, "bar")), want: newTestTuple(1, "bar").ToObject()}, {args: wrapArgs(Mul, newObject(ObjectType), newTestTuple(newObject(ObjectType))), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'object' and 'tuple'")}, {args: wrapArgs(Mul, NewTuple(newObject(ObjectType)), NewTuple()), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'tuple' and 'tuple'")}, {args: wrapArgs(Mul, NewTuple(None, None), MaxInt), wantExc: mustCreateException(OverflowErrorType, "result too large")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestTupleCompare(t *testing.T) { o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(NewTuple(), NewTuple()), want: compareAllResultEq}, {args: wrapArgs(newTestTuple("foo", o), newTestTuple("foo", o)), want: compareAllResultEq}, {args: wrapArgs(newTestTuple(4), newTestTuple(3, 0)), want: compareAllResultGT}, {args: wrapArgs(newTestTuple(4), newTestTuple(4, 3, 0)), want: compareAllResultLT}, {args: wrapArgs(NewTuple(o), NewTuple()), want: compareAllResultGT}, {args: wrapArgs(NewTuple(o), newTestTuple("foo")), want: compareAllResultLT}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestTupleCompareNotImplemented(t *testing.T) { cas := invokeTestCase{args: wrapArgs(NewTuple(), 3), want: NotImplemented} if err := runInvokeMethodTestCase(TupleType, "__eq__", &cas); err != "" { t.Error(err) } } func TestTupleContains(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestTuple("foo", 42, "bar"), 1), want: False.ToObject()}, {args: wrapArgs(newTestTuple("foo", 42, "bar"), "foo"), want: True.ToObject()}, {args: wrapArgs(newTestTuple("foo", 42, "bar"), 42), want: True.ToObject()}, {args: wrapArgs(newTestTuple("foo", 42, "bar"), "bar"), want: True.ToObject()}, {args: wrapArgs(NewTuple(), newTestSlice(50, 100)), want: False.ToObject()}, {args: wrapArgs(newTestTuple(1, 2, 3, 4, 5), newTestSlice(1, None, 2)), want: False.ToObject()}, {args: wrapArgs(NewTuple(), 1), want: False.ToObject()}, {args: wrapArgs(newTestTuple(32), -100), want: False.ToObject()}, {args: wrapArgs(newTestTuple(1, 2, 3), newTestSlice(1, None, 0)), want: False.ToObject()}, {args: wrapArgs(newTestTuple(true), None), want: False.ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(TupleType, "__contains__", &cas); err != "" { t.Error(err) } } } func TestTupleCount(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewTuple(), NewInt(1)), want: NewInt(0).ToObject()}, {args: wrapArgs(NewTuple(None, None, None), None), want: NewInt(3).ToObject()}, {args: wrapArgs(NewTuple()), wantExc: mustCreateException(TypeErrorType, "'count' of 'tuple' requires 2 arguments")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(TupleType, "count", &cas); err != "" { t.Error(err) } } } func BenchmarkTupleContains(b *testing.B) { b.Run("false-3", func(b *testing.B) { t := newTestTuple("foo", 42, "bar").ToObject() a := wrapArgs(1)[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) b.Run("false-10", func(b *testing.B) { t := newTestTuple("foo", 42, "bar", "foo", 42, "bar", "foo", 42, "bar", "baz").ToObject() a := wrapArgs(1)[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) b.Run("true-3.1", func(b *testing.B) { t := newTestTuple("foo", 42, "bar").ToObject() a := wrapArgs("foo")[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) b.Run("true-3.3", func(b *testing.B) { t := newTestTuple("foo", 42, "bar").ToObject() a := wrapArgs("bar")[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) b.Run("true-10.10", func(b *testing.B) { t := newTestTuple("foo", 42, "bar", "foo", 42, "bar", "foo", 42, "bar", "baz").ToObject() a := wrapArgs("baz")[0] f := NewRootFrame() b.ResetTimer() for i := 0; i < b.N; i++ { Contains(f, t, a) } }) } func TestTupleGetItem(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(newTestTuple("foo", 42, "bar"), 1), want: NewInt(42).ToObject()}, {args: wrapArgs(newTestTuple("foo", 42, "bar"), -3), want: NewStr("foo").ToObject()}, {args: wrapArgs(NewTuple(), newTestSlice(50, 100)), want: NewTuple().ToObject()}, {args: wrapArgs(newTestTuple(1, 2, 3, 4, 5), newTestSlice(1, None, 2)), want: newTestTuple(2, 4).ToObject()}, {args: wrapArgs(NewTuple(), 1), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(newTestTuple(32), -100), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(newTestTuple(1, 2, 3), newTestSlice(1, None, 0)), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, {args: wrapArgs(newTestTuple(true), None), wantExc: mustCreateException(TypeErrorType, "sequence indices must be integers, not NoneType")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(TupleType, "__getitem__", &cas); err != "" { t.Error(err) } } } func TestTupleLen(t *testing.T) { tuple := newTestTuple("foo", 42, "bar") if got := tuple.Len(); got != 3 { t.Errorf("%v.Len() = %v, want 3", tuple, got) } } func TestTupleNew(t *testing.T) { cases := []invokeTestCase{ {want: NewTuple().ToObject()}, {args: wrapArgs(newTestTuple(1, 2, 3)), want: newTestTuple(1, 2, 3).ToObject()}, {args: wrapArgs(newTestDict(1, "foo", "bar", None)), want: newTestTuple(1, "bar").ToObject()}, {args: wrapArgs(42), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable")}, } for _, cas := range cases { if err := runInvokeTestCase(TupleType.ToObject(), &cas); err != "" { t.Error(err) } } } func TestTupleStrRepr(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object) (*Tuple, *BaseException) { str, raised := ToStr(f, o) if raised != nil { return nil, raised } repr, raised := Repr(f, o) if raised != nil { return nil, raised } return newTestTuple(str, repr), nil }) cases := []invokeTestCase{ {args: wrapArgs(NewTuple()), want: newTestTuple("()", "()").ToObject()}, {args: wrapArgs(newTestTuple("foo")), want: newTestTuple("('foo',)", "('foo',)").ToObject()}, {args: wrapArgs(newTestTuple(TupleType, ExceptionType)), want: newTestTuple("(, )", "(, )").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestTupleIter(t *testing.T) { o := newObject(ObjectType) cases := []invokeTestCase{ {args: wrapArgs(NewTuple()), want: NewList().ToObject()}, {args: wrapArgs(newTestTuple(1, o, "foo")), want: newTestList(1, o, "foo").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(ListType.ToObject(), &cas); err != "" { t.Error(err) } } } func newTestTuple(elems ...interface{}) *Tuple { return NewTuple(wrapArgs(elems...)...) } ================================================ FILE: runtime/type.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" ) type typeFlag int const ( // Set when instances can be created via __new__. This is the default. // We need to be able to prohibit instantiation of certain internal // types like NoneType. CPython accomplishes this via tp_new == NULL but // we don't have a tp_new. typeFlagInstantiable typeFlag = 1 << iota // Set when the type can be used as a base class. This is the default. // Corresponds to the Py_TPFLAGS_BASETYPE flag in CPython. typeFlagBasetype typeFlag = 1 << iota typeFlagDefault = typeFlagInstantiable | typeFlagBasetype ) // Type represents Python 'type' objects. type Type struct { Object name string `attr:"__name__"` basis reflect.Type bases []*Type mro []*Type flags typeFlag slots typeSlots } var basisTypes = map[reflect.Type]*Type{ objectBasis: ObjectType, typeBasis: TypeType, } // newClass creates a Python type with the given name, base classes and type // dict. It is similar to the Python expression 'type(name, bases, dict)'. func newClass(f *Frame, meta *Type, name string, bases []*Type, dict *Dict) (*Type, *BaseException) { numBases := len(bases) if numBases == 0 { return nil, f.RaiseType(TypeErrorType, "class must have base classes") } var basis reflect.Type for _, base := range bases { if base.flags&typeFlagBasetype == 0 { format := "type '%s' is not an acceptable base type" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, base.Name())) } basis = basisSelect(basis, base.basis) } if basis == nil { return nil, f.RaiseType(TypeErrorType, "class layout error") } t := newType(meta, name, basis, bases, dict) // Populate slots for any special methods overridden in dict. slotsValue := reflect.ValueOf(&t.slots).Elem() for i := 0; i < numSlots; i++ { dictFunc, raised := dict.GetItemString(f, slotNames[i]) if raised != nil { return nil, raised } if dictFunc != nil { slotField := slotsValue.Field(i) slotValue := reflect.New(slotField.Type().Elem()) if slotValue.Interface().(slot).wrapCallable(dictFunc) { slotField.Set(slotValue) } } } if err := prepareType(t); err != "" { return nil, f.RaiseType(TypeErrorType, err) } // Set the __module__ attr if it's not already specified. mod, raised := dict.GetItemString(f, "__module__") if raised != nil { return nil, raised } if mod == nil { if raised := dict.SetItemString(f, "__module__", builtinStr.ToObject()); raised != nil { return nil, raised } } return t, nil } func newType(meta *Type, name string, basis reflect.Type, bases []*Type, dict *Dict) *Type { return &Type{ Object: Object{typ: meta, dict: dict}, name: name, basis: basis, bases: bases, flags: typeFlagDefault, } } func newBasisType(name string, basis reflect.Type, basisFunc interface{}, base *Type) *Type { if _, ok := basisTypes[basis]; ok { logFatal(fmt.Sprintf("type for basis already exists: %s", basis)) } if basis.Kind() != reflect.Struct { logFatal(fmt.Sprintf("basis must be a struct not: %s", basis.Kind())) } if basis.NumField() == 0 { logFatal(fmt.Sprintf("1st field of basis must be base type's basis")) } if basis.Field(0).Type != base.basis { logFatal(fmt.Sprintf("1st field of basis must be base type's basis not: %s", basis.Field(0).Type)) } basisFuncValue := reflect.ValueOf(basisFunc) basisFuncType := basisFuncValue.Type() if basisFuncValue.Kind() != reflect.Func || basisFuncType.NumIn() != 1 || basisFuncType.NumOut() != 1 || basisFuncType.In(0) != reflect.PtrTo(objectBasis) || basisFuncType.Out(0) != reflect.PtrTo(basis) { logFatal(fmt.Sprintf("expected basis func of type func(*Object) *%s", nativeTypeName(basis))) } t := newType(TypeType, name, basis, []*Type{base}, nil) t.slots.Basis = &basisSlot{func(o *Object) reflect.Value { return basisFuncValue.Call([]reflect.Value{reflect.ValueOf(o)})[0].Elem() }} basisTypes[basis] = t return t } func newSimpleType(name string, base *Type) *Type { return newType(TypeType, name, base.basis, []*Type{base}, nil) } // prepareBuiltinType initializes the builtin typ by populating dict with // struct field descriptors and slot wrappers, and then calling prepareType. func prepareBuiltinType(typ *Type, init builtinTypeInit) { dict := map[string]*Object{"__module__": builtinStr.ToObject()} if init != nil { init(dict) } // For basis types, export field descriptors. if basis := typ.basis; basisTypes[basis] == typ { numFields := basis.NumField() for i := 0; i < numFields; i++ { field := basis.Field(i) if attr := field.Tag.Get("attr"); attr != "" { fieldMode := fieldDescriptorRO if mode := field.Tag.Get("attr_mode"); mode == "rw" { fieldMode = fieldDescriptorRW } dict[attr] = makeStructFieldDescriptor(typ, field.Name, attr, fieldMode) } } } // Create dict entries for slot methods. slotsValue := reflect.ValueOf(&typ.slots).Elem() for i := 0; i < numSlots; i++ { slotField := slotsValue.Field(i) if !slotField.IsNil() { slot := slotField.Interface().(slot) if fun := slot.makeCallable(typ, slotNames[i]); fun != nil { dict[slotNames[i]] = fun } } } typ.setDict(newStringDict(dict)) if err := prepareType(typ); err != "" { logFatal(err) } } // prepareType calculates typ's mro and inherits its flags and slots from its // base classes. func prepareType(typ *Type) string { typ.mro = mroCalc(typ) if typ.mro == nil { return fmt.Sprintf("mro error for: %s", typ.name) } for _, base := range typ.mro { if base.flags&typeFlagInstantiable == 0 { typ.flags &^= typeFlagInstantiable } if base.flags&typeFlagBasetype == 0 { typ.flags &^= typeFlagBasetype } } // Inherit slots from typ's mro. slotsValue := reflect.ValueOf(&typ.slots).Elem() for i := 0; i < numSlots; i++ { slotField := slotsValue.Field(i) if slotField.IsNil() { for _, base := range typ.mro { baseSlotFunc := reflect.ValueOf(base.slots).Field(i) if !baseSlotFunc.IsNil() { slotField.Set(baseSlotFunc) break } } } } return "" } // Precondition: At least one of seqs is non-empty. func mroMerge(seqs [][]*Type) []*Type { var res []*Type numSeqs := len(seqs) hasNonEmptySeqs := true for hasNonEmptySeqs { var cand *Type for i := 0; i < numSeqs && cand == nil; i++ { // The next candidate will be absent from or at the head // of all lists. If we try a candidate and we find it's // somewhere past the head of one of the lists, reject. seq := seqs[i] if len(seq) == 0 { continue } cand = seq[0] RejectCandidate: for _, seq := range seqs { numElems := len(seq) for j := 1; j < numElems; j++ { if seq[j] == cand { cand = nil break RejectCandidate } } } } if cand == nil { // We could not find a candidate meaning that the // hierarchy is inconsistent. return nil } res = append(res, cand) hasNonEmptySeqs = false for i, seq := range seqs { if len(seq) > 0 { if seq[0] == cand { // Remove the candidate from all lists // (it will only be found at the head of // any list because otherwise it would // have been rejected above.) seqs[i] = seq[1:] } if len(seqs[i]) > 0 { hasNonEmptySeqs = true } } } } return res } func mroCalc(t *Type) []*Type { seqs := [][]*Type{{t}} for _, b := range t.bases { seqs = append(seqs, b.mro) } seqs = append(seqs, t.bases) return mroMerge(seqs) } func toTypeUnsafe(o *Object) *Type { return (*Type)(o.toPointer()) } // ToObject upcasts t to an Object. func (t *Type) ToObject() *Object { return &t.Object } // Name returns t's name field. func (t *Type) Name() string { return t.name } // FullName returns t's fully qualified name including the module. func (t *Type) FullName(f *Frame) (string, *BaseException) { moduleAttr, raised := t.Dict().GetItemString(f, "__module__") if raised != nil { return "", raised } if moduleAttr == nil { return t.Name(), nil } if moduleAttr.isInstance(StrType) { if s := toStrUnsafe(moduleAttr).Value(); s != "__builtin__" { return fmt.Sprintf("%s.%s", s, t.Name()), nil } } return t.Name(), nil } func (t *Type) isSubclass(super *Type) bool { for _, b := range t.mro { if b == super { return true } } return false } func (t *Type) mroLookup(f *Frame, name *Str) (*Object, *BaseException) { for _, t := range t.mro { v, raised := t.Dict().GetItem(f, name.ToObject()) if v != nil || raised != nil { return v, raised } } return nil, nil } var typeBasis = reflect.TypeOf(Type{}) func typeBasisFunc(o *Object) reflect.Value { return reflect.ValueOf(toTypeUnsafe(o)).Elem() } // TypeType is the object representing the Python 'type' type. // // Don't use newType() since that depends on the initialization of // TypeType. var TypeType = &Type{ name: "type", basis: typeBasis, bases: []*Type{ObjectType}, flags: typeFlagDefault, slots: typeSlots{Basis: &basisSlot{typeBasisFunc}}, } func typeCall(f *Frame, callable *Object, args Args, kwargs KWArgs) (*Object, *BaseException) { t := toTypeUnsafe(callable) newFunc := t.slots.New if newFunc == nil { return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("type %s has no __new__", t.Name())) } o, raised := newFunc.Fn(f, t, args, kwargs) if raised != nil { return nil, raised } if init := o.Type().slots.Init; init != nil { if _, raised := init.Fn(f, o, args, kwargs); raised != nil { return nil, raised } } return o, nil } // typeGetAttribute is very similar to objectGetAttribute except that it uses // MRO to resolve dict attributes rather than just the type's own dict and the // exception message is slightly different. func typeGetAttribute(f *Frame, o *Object, name *Str) (*Object, *BaseException) { t := toTypeUnsafe(o) // Look for a data descriptor in the metaclass. var metaGet *getSlot metaType := t.typ metaAttr, raised := metaType.mroLookup(f, name) if raised != nil { return nil, raised } if metaAttr != nil { metaGet = metaAttr.typ.slots.Get if metaGet != nil && (metaAttr.typ.slots.Set != nil || metaAttr.typ.slots.Delete != nil) { return metaGet.Fn(f, metaAttr, t.ToObject(), metaType) } } // Look in dict of this type and its bases. attr, raised := t.mroLookup(f, name) if raised != nil { return nil, raised } if attr != nil { if get := attr.typ.slots.Get; get != nil { return get.Fn(f, attr, None, t) } return attr, nil } // Use the (non-data) descriptor from the metaclass. if metaGet != nil { return metaGet.Fn(f, metaAttr, t.ToObject(), metaType) } // Return the ordinary attr from metaclass. if metaAttr != nil { return metaAttr, nil } msg := fmt.Sprintf("type object '%s' has no attribute '%s'", t.Name(), name.Value()) return nil, f.RaiseType(AttributeErrorType, msg) } func typeNew(f *Frame, t *Type, args Args, kwargs KWArgs) (*Object, *BaseException) { switch len(args) { case 0: return nil, f.RaiseType(TypeErrorType, "type() takes 1 or 3 arguments") case 1: return args[0].typ.ToObject(), nil } // case 3+ if raised := checkMethodArgs(f, "__new__", args, StrType, TupleType, DictType); raised != nil { return nil, raised } name := toStrUnsafe(args[0]).Value() bases := toTupleUnsafe(args[1]).elems dict := toDictUnsafe(args[2]) baseTypes := make([]*Type, len(bases)) meta := t for i, o := range bases { if !o.isInstance(TypeType) { s, raised := Repr(f, o) if raised != nil { return nil, raised } return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("not a valid base class: %s", s.Value())) } // Choose the most derived metaclass among all the bases to be // the metaclass for the new type. if o.typ.isSubclass(meta) { meta = o.typ } else if !meta.isSubclass(o.typ) { msg := "metaclass conflict: the metaclass of a derived class must " + "a be a (non-strict) subclass of the metaclasses of all its bases" return nil, f.RaiseType(TypeErrorType, msg) } baseTypes[i] = toTypeUnsafe(o) } ret, raised := newClass(f, meta, name, baseTypes, dict) if raised != nil { return nil, raised } return ret.ToObject(), nil } func typeRepr(f *Frame, o *Object) (*Object, *BaseException) { s, raised := toTypeUnsafe(o).FullName(f) if raised != nil { return nil, raised } return NewStr(fmt.Sprintf("", s)).ToObject(), nil } func initTypeType(map[string]*Object) { TypeType.typ = TypeType TypeType.slots.Call = &callSlot{typeCall} TypeType.slots.GetAttribute = &getAttributeSlot{typeGetAttribute} TypeType.slots.New = &newSlot{typeNew} TypeType.slots.Repr = &unaryOpSlot{typeRepr} } // basisParent returns the immediate ancestor of basis, which is its first // field. Returns nil when basis is objectBasis (the root of basis hierarchy.) func basisParent(basis reflect.Type) reflect.Type { if basis == objectBasis { return nil } return basis.Field(0).Type } // basisSelect returns b1 if b2 inherits from it, b2 if b1 inherits from b2, // otherwise nil. b1 can be nil in which case b2 is always returned. func basisSelect(b1, b2 reflect.Type) reflect.Type { if b1 == nil { return b2 } // Search up b1's inheritance chain to see if b2 is present. basis := b1 for basis != nil && basis != b2 { basis = basisParent(basis) } if basis != nil { return b1 } // Search up b2's inheritance chain to see if b1 is present. basis = b2 for basis != nil && basis != b1 { basis = basisParent(basis) } if basis != nil { return b2 } return nil } ================================================ FILE: runtime/type_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" "strings" "testing" ) func TestNewClass(t *testing.T) { type strBasisStruct struct{ Str } strBasisStructFunc := func(o *Object) *strBasisStruct { return (*strBasisStruct)(o.toPointer()) } fooType := newBasisType("Foo", reflect.TypeOf(strBasisStruct{}), strBasisStructFunc, StrType) defer delete(basisTypes, fooType.basis) fooType.setDict(NewDict()) prepareType(fooType) cases := []struct { wantBasis reflect.Type invokeTestCase }{ {objectBasis, invokeTestCase{args: wrapArgs([]*Type{ObjectType}), want: None}}, {fooType.basis, invokeTestCase{args: wrapArgs([]*Type{fooType, StrType}), want: None}}, {fooType.basis, invokeTestCase{args: wrapArgs([]*Type{fooType, StrType, ObjectType}), want: None}}, {nil, invokeTestCase{args: wrapArgs([]*Type{}), wantExc: mustCreateException(TypeErrorType, "class must have base classes")}}, {nil, invokeTestCase{args: wrapArgs([]*Type{BoolType, ObjectType}), wantExc: mustCreateException(TypeErrorType, "type 'bool' is not an acceptable base type")}}, {nil, invokeTestCase{args: wrapArgs([]*Type{IntType, StrType}), wantExc: mustCreateException(TypeErrorType, "class layout error")}}, {nil, invokeTestCase{args: wrapArgs([]*Type{StrType, fooType}), wantExc: mustCreateException(TypeErrorType, "mro error for: Foo")}}, } for _, cas := range cases { fun := wrapFuncForTest(func(f *Frame, bases []*Type) *BaseException { cls, raised := newClass(f, TypeType, "Foo", bases, NewDict()) if raised != nil { return raised } if cls.basis != cas.wantBasis { t.Errorf("type('Foo', %v, {}) had basis %v, want %v", bases, cls.basis, cas.wantBasis) } return nil }) if err := runInvokeTestCase(fun, &cas.invokeTestCase); err != "" { t.Error(err) } } } func TestNewBasisType(t *testing.T) { type basisStruct struct{ Object } basisStructFunc := func(o *Object) *basisStruct { return (*basisStruct)(o.toPointer()) } basis := reflect.TypeOf(basisStruct{}) typ := newBasisType("Foo", basis, basisStructFunc, ObjectType) defer delete(basisTypes, basis) if typ.Type() != TypeType { t.Errorf("got %q, want a type", typ.Type().Name()) } if typ.Dict() != nil { t.Error("type's dict was expected to be nil") } wantBases := []*Type{ObjectType} if !reflect.DeepEqual(wantBases, typ.bases) { t.Errorf("typ.bases = %v, want %v, ", typ.bases, wantBases) } if typ.mro != nil { t.Errorf("type's mro expected to be nil, got %v", typ.mro) } if name := typ.Name(); name != "Foo" { t.Errorf(`Foo.Name() = %q, want "Foo"`, name) } foo := (*basisStruct)(newObject(typ).toPointer()) if typ.slots.Basis == nil { t.Error("type's Basis slot is nil") } else if got := typ.slots.Basis.Fn(&foo.Object); got.Type() != basis || got.Addr().Interface().(*basisStruct) != foo { t.Errorf("Foo.__basis__(%v) = %v, want %v", &foo.Object, got, foo) } } func TestNewSimpleType(t *testing.T) { got := newSimpleType("Foo", ObjectType) if got.Object.typ != TypeType { t.Errorf(`newSimpleType got %q, want "type"`, got.Type().Name()) } if got.basis != objectBasis { t.Errorf("newSimpleType result got basis %v, want %v", got.basis, objectBasis) } wantBases := []*Type{ObjectType} if !reflect.DeepEqual(got.bases, wantBases) { t.Errorf("newSimpleType got bases %v, want %v", got.bases, wantBases) } if name := got.Name(); name != "Foo" { t.Errorf(`Foo.Name() = %q, want "Foo"`, name) } } func TestInvalidBasisType(t *testing.T) { type intFieldStruct struct{ int } type emptyStruct struct{} type objectBasisStruct struct{ Object } oldLogFatal := logFatal defer func() { logFatal = oldLogFatal }() logFatal = func(msg string) { panic(msg) } cases := []struct { basis reflect.Type basisFunc interface{} wantMsg string }{ {objectBasis, objectBasisFunc, "basis already exists"}, {reflect.TypeOf(int(0)), objectBasisFunc, "basis must be a struct"}, {reflect.TypeOf(emptyStruct{}), objectBasisFunc, "1st field of basis must be base type's basis"}, {reflect.TypeOf(intFieldStruct{}), objectBasisFunc, "1st field of basis must be base type's basis not: int"}, {reflect.TypeOf(objectBasisStruct{}), objectBasisFunc, "expected basis func of type func(*Object) *objectBasisStruct"}, } for _, cas := range cases { func() { defer func() { if msg, ok := recover().(string); !ok || !strings.Contains(msg, cas.wantMsg) { t.Errorf("logFatal() called with %q, want error like %q", msg, cas.wantMsg) } }() newBasisType("Foo", cas.basis, cas.basisFunc, ObjectType) }() } } func TestPrepareType(t *testing.T) { type objectBasisStruct struct{ Object } objectBasisStructFunc := func(o *Object) *objectBasisStruct { return (*objectBasisStruct)(o.toPointer()) } type strBasisStruct struct{ Str } strBasisStructFunc := func(o *Object) *strBasisStruct { return (*strBasisStruct)(o.toPointer()) } cases := []struct { basis reflect.Type basisFunc interface{} base *Type wantMro []*Type }{ {reflect.TypeOf(objectBasisStruct{}), objectBasisStructFunc, ObjectType, []*Type{nil, ObjectType}}, {reflect.TypeOf(strBasisStruct{}), strBasisStructFunc, StrType, []*Type{nil, StrType, BaseStringType, ObjectType}}, } for _, cas := range cases { typ := newBasisType("Foo", cas.basis, cas.basisFunc, cas.base) defer delete(basisTypes, cas.basis) typ.setDict(NewDict()) prepareType(typ) cas.wantMro[0] = typ if !reflect.DeepEqual(typ.mro, cas.wantMro) { t.Errorf("typ.mro = %v, want %v", typ.mro, cas.wantMro) } } } func makeTestType(name string, bases ...*Type) *Type { return newType(TypeType, name, nil, bases, NewDict()) } func TestMroCalc(t *testing.T) { fooType := makeTestType("Foo", ObjectType) barType := makeTestType("Bar", StrType, fooType) bazType := makeTestType("Baz", fooType, StrType) // Boo has an inconsistent hierarchy since it's not possible to order // mro such that StrType is before fooType and fooType is also before // StrType. booType := makeTestType("Boo", barType, bazType) cases := []struct { typ *Type wantMro []*Type }{ {fooType, []*Type{fooType, ObjectType}}, {barType, []*Type{barType, StrType, BaseStringType, fooType, ObjectType}}, {bazType, []*Type{bazType, fooType, StrType, BaseStringType, ObjectType}}, {booType, nil}, } for _, cas := range cases { cas.typ.mro = mroCalc(cas.typ) if !reflect.DeepEqual(cas.wantMro, cas.typ.mro) { t.Errorf("%s.mro = %v, want %v", cas.typ.Name(), cas.typ.mro, cas.wantMro) } } } func TestTypeIsSubclass(t *testing.T) { fooType := makeTestType("Foo", ObjectType) prepareType(fooType) barType := makeTestType("Bar", StrType, fooType) prepareType(barType) cases := []struct { typ *Type super *Type want bool }{ {fooType, ObjectType, true}, {fooType, StrType, false}, {barType, ObjectType, true}, {barType, fooType, true}, {barType, StrType, true}, {barType, TypeType, false}, } for _, cas := range cases { got := cas.typ.isSubclass(cas.super) if got != cas.want { t.Errorf("%s.isSubclass(%s) = %v, want %v", cas.typ.Name(), cas.super.Name(), got, cas.want) } } } func TestTypeCall(t *testing.T) { fooType := makeTestType("Foo") prepareType(fooType) emptyExc := toBaseExceptionUnsafe(newObject(ExceptionType)) emptyExc.args = NewTuple() cases := []invokeTestCase{ {wantExc: mustCreateException(TypeErrorType, "unbound method __call__() must be called with type instance as first argument (got nothing instead)")}, {args: wrapArgs(42), wantExc: mustCreateException(TypeErrorType, "unbound method __call__() must be called with type instance as first argument (got int instance instead)")}, {args: wrapArgs(fooType), wantExc: mustCreateException(TypeErrorType, "type Foo has no __new__")}, {args: wrapArgs(IntType), want: NewInt(0).ToObject()}, {args: wrapArgs(ExceptionType, "blah"), want: mustCreateException(ExceptionType, "blah").ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(TypeType, "__call__", &cas); err != "" { t.Error(err) } } } func TestNewWithSubclass(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(StrType, "abc"), want: None}, {args: wrapArgs(IntType, 3), want: None}, {args: wrapArgs(UnicodeType, "abc"), want: None}, } simpleRepr := newBuiltinFunction("__repr__", func(_ *Frame, args Args, _ KWArgs) (*Object, *BaseException) { return NewStr(fmt.Sprintf("%s object", args[0].typ.Name())).ToObject(), nil }).ToObject() constantFunc := func(name string, value *Object) *Object { return newBuiltinFunction(name, func(_ *Frame, _ Args, _ KWArgs) (*Object, *BaseException) { return value, nil }).ToObject() } for _, cas := range cases { fun := wrapFuncForTest(func(f *Frame, basisType *Type, o *Object) *BaseException { subclassTypeName := "SubclassOf" + basisType.Name() // Create a subclass of the basis type. subclassType := newTestClass(subclassTypeName, []*Type{basisType}, newStringDict(map[string]*Object{ "__repr__": simpleRepr, })) subclassObject, raised := subclassType.Call(f, Args{o}, nil) if raised != nil { return raised } slotName := "__" + basisType.Name() + "__" fooType := newTestClass("FooFor"+basisType.Name(), []*Type{ObjectType}, newStringDict(map[string]*Object{ slotName: constantFunc(slotName, subclassObject), "__repr__": simpleRepr, })) foo := newObject(fooType) // Test that (subclassObject) returns an object of the basis type, not the subclass. got, raised := basisType.Call(f, Args{subclassObject}, nil) if raised != nil { return raised } if got.typ != basisType { t.Errorf("type(%s(%s)) = %s, want %s", basisType.Name(), subclassObject, got.typ.Name(), basisType.Name()) } // Test that subclass objects returned from ____ slots are left intact. got, raised = basisType.Call(f, Args{foo}, nil) if raised != nil { return raised } if got.typ != subclassType { t.Errorf("type(%s(%s)) = %s, want %s", basisType.Name(), foo, got.typ.Name(), basisType.Name()) } return nil }) if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestTypeGetAttribute(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object, name *Str) (*Object, *BaseException) { return GetAttr(f, o, name, nil) }) // class Getter(object): // def __get__(self, *args): // return "got getter" getterType := newTestClass("Getter", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__get__": newBuiltinFunction("__get__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("got getter").ToObject(), nil }).ToObject(), })) getter := newObject(getterType) // class Setter(object): // def __get__(self, *args): // return "got setter" // def __set__(self, *args): // pass setterType := newTestClass("Setter", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__get__": newBuiltinFunction("__get__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("got setter").ToObject(), nil }).ToObject(), "__set__": newBuiltinFunction("__set__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return None, nil }).ToObject(), })) setter := newObject(setterType) // class Foo(object): // pass fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "bar": NewInt(42).ToObject(), "baz": NewStr("Foo's baz").ToObject(), "foogetter": getter, })) // class BarMeta(type): // pass barMetaType := newTestClass("BarMeta", []*Type{TypeType}, newStringDict(map[string]*Object{ "bar": NewStr("BarMeta's bar").ToObject(), "boo": NewInt(123).ToObject(), "barmetagetter": getter, "barmetasetter": setter, })) // class Bar(Foo): // __metaclass__ = BarMeta // bar = Bar() barType := &Type{Object: Object{typ: barMetaType}, name: "Bar", basis: fooType.basis, bases: []*Type{fooType}} barType.setDict(newTestDict("bar", "Bar's bar", "foo", 101, "barsetter", setter, "barmetasetter", "NOT setter")) bar := newObject(barType) prepareType(barType) cases := []invokeTestCase{ {args: wrapArgs(fooType, "bar"), want: NewInt(42).ToObject()}, {args: wrapArgs(fooType, "baz"), want: NewStr("Foo's baz").ToObject()}, {args: wrapArgs(barMetaType, "barmetagetter"), want: NewStr("got getter").ToObject()}, {args: wrapArgs(barType, "bar"), want: NewStr("Bar's bar").ToObject()}, {args: wrapArgs(barType, "baz"), want: NewStr("Foo's baz").ToObject()}, {args: wrapArgs(barType, "foo"), want: NewInt(101).ToObject()}, {args: wrapArgs(barType, "barmetagetter"), want: NewStr("got getter").ToObject()}, {args: wrapArgs(barType, "barmetasetter"), want: NewStr("got setter").ToObject()}, {args: wrapArgs(barType, "boo"), want: NewInt(123).ToObject()}, {args: wrapArgs(bar, "boo"), wantExc: mustCreateException(AttributeErrorType, "'Bar' object has no attribute 'boo'")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestTypeName(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, NewDict()) fun := wrapFuncForTest(func(f *Frame, t *Type) (*Object, *BaseException) { return GetAttr(f, t.ToObject(), internedName, nil) }) cas := invokeTestCase{args: wrapArgs(fooType), want: NewStr("Foo").ToObject()} if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } func TestTypeNew(t *testing.T) { fooMetaType := newTestClass("FooMeta", []*Type{TypeType}, NewDict()) fooType, raised := newClass(NewRootFrame(), fooMetaType, "Foo", []*Type{ObjectType}, NewDict()) if raised != nil { panic(raised) } barMetaType := newTestClass("BarMeta", []*Type{TypeType}, NewDict()) barType, raised := newClass(NewRootFrame(), barMetaType, "Bar", []*Type{ObjectType}, NewDict()) if raised != nil { panic(raised) } var bazMetaType *Type bazMetaType = newTestClass("BazMeta", []*Type{barMetaType}, newStringDict(map[string]*Object{ // Returns true if type(lhs) == rhs. "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__eq__", args, TypeType, TypeType); raised != nil { return nil, raised } return GetBool(args[0].typ == toTypeUnsafe(args[1])).ToObject(), nil }).ToObject(), })) bazType, raised := newClass(NewRootFrame(), bazMetaType, "Baz", []*Type{ObjectType}, NewDict()) if raised != nil { panic(raised) } cases := []invokeTestCase{ {wantExc: mustCreateException(TypeErrorType, "'__new__' requires 1 arguments")}, {args: wrapArgs(TypeType), wantExc: mustCreateException(TypeErrorType, "type() takes 1 or 3 arguments")}, {args: wrapArgs(TypeType, "foo", newTestTuple(false), NewDict()), wantExc: mustCreateException(TypeErrorType, "not a valid base class: False")}, {args: wrapArgs(TypeType, None), want: NoneType.ToObject()}, {args: wrapArgs(fooMetaType, "Qux", newTestTuple(fooType, barType), NewDict()), wantExc: mustCreateException(TypeErrorType, "metaclass conflict: the metaclass of a derived class must a be a (non-strict) subclass of the metaclasses of all its bases")}, // Test that the metaclass of the result is the most derived // metaclass of the bases. In this case that should be // bazMetaType so pass bazMetaType to be compared by the __eq__ // operator defined above. {args: wrapArgs(barMetaType, "Qux", newTestTuple(barType, bazType), NewDict()), want: bazMetaType.ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(TypeType, "__new__", &cas); err != "" { t.Error(err) } } } func TestTypeNewResult(t *testing.T) { fooType := makeTestType("Foo", ObjectType) prepareType(fooType) fun := wrapFuncForTest(func(f *Frame) *BaseException { newFunc, raised := GetAttr(f, TypeType.ToObject(), NewStr("__new__"), nil) if raised != nil { return raised } ret, raised := newFunc.Call(f, wrapArgs(TypeType, "Bar", newTestTuple(fooType, StrType), NewDict()), nil) if raised != nil { return raised } if !ret.isInstance(TypeType) { t.Errorf("type('Bar', (Foo, str), {}) = %v, want type instance", ret) } else if typ := toTypeUnsafe(ret); typ.basis != StrType.basis { t.Errorf("type('Bar', (Foo, str), {}) basis is %v, want %v", typ.basis, StrType.basis) } else if wantMro := []*Type{typ, fooType, StrType, BaseStringType, ObjectType}; !reflect.DeepEqual(typ.mro, wantMro) { t.Errorf("type('Bar', (Foo, str), {}).__mro__ = %v, want %v", typ.mro, wantMro) } return nil }) if err := runInvokeTestCase(fun, &invokeTestCase{want: None}); err != "" { t.Error(err) } } func TestTypeStrRepr(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, o *Object) (*Tuple, *BaseException) { str, raised := ToStr(f, o) if raised != nil { return nil, raised } repr, raised := Repr(f, o) if raised != nil { return nil, raised } return newTestTuple(str, repr), nil }) fooType := newTestClass("Foo", []*Type{ObjectType}, newTestDict("__module__", "foo.bar")) cases := []invokeTestCase{ {args: wrapArgs(TypeErrorType), want: newTestTuple("", "").ToObject()}, {args: wrapArgs(TupleType), want: newTestTuple("", "").ToObject()}, {args: wrapArgs(TypeType), want: newTestTuple("", "").ToObject()}, {args: wrapArgs(fooType), want: newTestTuple("", "").ToObject()}, {args: wrapArgs(mustNotRaise(WrapNative(NewRootFrame(), reflect.ValueOf(t))).Type()), want: newTestTuple("", "").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestTypeModule(t *testing.T) { fn := newBuiltinFunction("__module__", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "__module__", args, TypeType); raised != nil { return nil, raised } mod, raised := toTypeUnsafe(args[0]).Dict().GetItemString(f, "__module__") if raised != nil || mod != nil { return mod, raised } return None, nil }).ToObject() fooType := newTestClass("Foo", []*Type{ObjectType}, newTestDict("__module__", "foo.bar")) barType := newTestClass("Bar", []*Type{ObjectType}, NewDict()) cases := []invokeTestCase{ {args: wrapArgs(IntType), want: NewStr("__builtin__").ToObject()}, {args: wrapArgs(mustNotRaise(WrapNative(NewRootFrame(), reflect.ValueOf(t))).Type()), want: NewStr("__builtin__").ToObject()}, {args: wrapArgs(fooType), want: NewStr("foo.bar").ToObject()}, {args: wrapArgs(barType), want: NewStr("__builtin__").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fn, &cas); err != "" { t.Error(err) } } } func newTestClass(name string, bases []*Type, dict *Dict) *Type { t, raised := newClass(NewRootFrame(), TypeType, name, bases, dict) if raised != nil { panic(raised) } return t } // newTestClassStrictEq returns a new class that defines eq and ne operators // that check whether the lhs and rhs have the same type and that the value // fields are also equal. This is useful for testing that the builtin types // return objects of the correct type for their __new__ method. func newTestClassStrictEq(name string, base *Type) *Type { var t *Type t = newTestClass(name, []*Type{base}, newStringDict(map[string]*Object{ "__repr__": newBuiltinFunction("__repr__", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__repr__", args, t); raised != nil { return nil, raised } repr, raised := GetAttr(f, base.ToObject(), NewStr("__repr__"), nil) if raised != nil { return nil, raised } s, raised := repr.Call(f, Args{args[0]}, nil) if raised != nil { return nil, raised } if !s.isInstance(StrType) { return nil, f.RaiseType(TypeErrorType, "__repr__ returned non-str") } return NewStr(fmt.Sprintf("%s(%s)", t.Name(), toStrUnsafe(s).Value())).ToObject(), nil }).ToObject(), "__eq__": newBuiltinFunction("__eq__", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__eq__", args, t, ObjectType); raised != nil { return nil, raised } if args[1].typ != t { return False.ToObject(), nil } return base.slots.Eq.Fn(f, args[0], args[1]) }).ToObject(), "__ne__": newBuiltinFunction("__ne__", func(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__ne__", args, t, ObjectType); raised != nil { return nil, raised } o, raised := Eq(f, args[0], args[1]) if raised != nil { return nil, raised } eq, raised := IsTrue(f, o) if raised != nil { return nil, raised } return GetBool(eq).ToObject(), nil }).ToObject(), })) return t } ================================================ FILE: runtime/unicode.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "bytes" "fmt" "reflect" "unicode" "unicode/utf8" ) var ( // UnicodeType is the object representing the Python 'unicode' type. UnicodeType = newBasisType("unicode", reflect.TypeOf(Unicode{}), toUnicodeUnsafe, BaseStringType) ) // Unicode represents Python 'unicode' objects. The string value is stored as // utf-32 data. type Unicode struct { Object value []rune } // NewUnicode returns a new Unicode holding the given string value. value is // assumed to be a valid utf-8 string. func NewUnicode(value string) *Unicode { return NewUnicodeFromRunes(bytes.Runes([]byte(value))) } // NewUnicodeFromRunes returns a new Unicode holding the given runes. func NewUnicodeFromRunes(value []rune) *Unicode { return &Unicode{Object{typ: UnicodeType}, value} } func toUnicodeUnsafe(o *Object) *Unicode { return (*Unicode)(o.toPointer()) } // Encode translates the runes in s into a str with the given encoding. // // NOTE: If s contains surrogates (e.g. U+D800), Encode will raise // UnicodeDecodeError consistent with CPython 3.x but different than 2.x. func (s *Unicode) Encode(f *Frame, encoding, errors string) (*Str, *BaseException) { // TODO: Support custom encodings and error handlers. normalized := normalizeEncoding(encoding) if normalized != "utf8" { return nil, f.RaiseType(LookupErrorType, fmt.Sprintf("unknown encoding: %s", encoding)) } buf := bytes.Buffer{} for i, r := range s.Value() { switch { case utf8.ValidRune(r): buf.WriteRune(r) case errors == EncodeIgnore: // Do nothing case errors == EncodeReplace: buf.WriteRune(unicode.ReplacementChar) case errors == EncodeStrict: format := "'%s' codec can't encode character %s in position %d" return nil, f.RaiseType(UnicodeEncodeErrorType, fmt.Sprintf(format, encoding, escapeRune(r), i)) default: format := "unknown error handler name '%s'" return nil, f.RaiseType(LookupErrorType, fmt.Sprintf(format, errors)) } } return NewStr(buf.String()), nil } // ToObject upcasts s to an Object. func (s *Unicode) ToObject() *Object { return &s.Object } // Value returns the underlying string value held by s. func (s *Unicode) Value() []rune { return s.value } func unicodeAdd(f *Frame, v, w *Object) (*Object, *BaseException) { unicodeV := toUnicodeUnsafe(v) unicodeW, raised := unicodeCoerce(f, w) if raised != nil { return nil, raised } lenV := len(unicodeV.Value()) newLen := lenV + len(unicodeW.Value()) if newLen < 0 { return nil, f.RaiseType(OverflowErrorType, errResultTooLarge) } value := make([]rune, newLen) copy(value, unicodeV.Value()) copy(value[lenV:], unicodeW.Value()) return NewUnicodeFromRunes(value).ToObject(), nil } func unicodeContains(f *Frame, o *Object, value *Object) (*Object, *BaseException) { lhs := toUnicodeUnsafe(o).Value() s, raised := unicodeCoerce(f, value) if raised != nil { return nil, raised } rhs := s.Value() lhsLen, rhsLen := len(lhs), len(rhs) maxOffset := lhsLen - rhsLen for offset := 0; offset <= maxOffset; offset++ { if runeSliceCmp(lhs[offset:offset+rhsLen], rhs) == 0 { return True.ToObject(), nil } } return False.ToObject(), nil } func unicodeEncode(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { // TODO: Accept unicode for encoding and errors args. expectedTypes := []*Type{UnicodeType, StrType, StrType} argc := len(args) if argc >= 1 && argc < 3 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "encode", args, expectedTypes...); raised != nil { return nil, raised } encoding := EncodeDefault if argc > 1 { encoding = toStrUnsafe(args[1]).Value() } errors := EncodeStrict if argc > 2 { errors = toStrUnsafe(args[2]).Value() } ret, raised := toUnicodeUnsafe(args[0]).Encode(f, encoding, errors) if raised != nil { return nil, raised } return ret.ToObject(), nil } func unicodeEq(f *Frame, v, w *Object) (*Object, *BaseException) { return unicodeCompareEq(f, toUnicodeUnsafe(v), w, true) } func unicodeGE(f *Frame, v, w *Object) (*Object, *BaseException) { return unicodeCompare(f, toUnicodeUnsafe(v), w, False, True, True) } // unicodeGetItem returns a slice of string depending on whether index is an // integer or a slice. If index is neither of those types then a TypeError is // returned. func unicodeGetItem(f *Frame, o, key *Object) (*Object, *BaseException) { s := toUnicodeUnsafe(o).Value() switch { case key.typ.slots.Index != nil: index, raised := seqCheckedIndex(f, len(s), toIntUnsafe(key).Value()) if raised != nil { return nil, raised } return NewUnicodeFromRunes([]rune{s[index]}).ToObject(), nil case key.isInstance(SliceType): slice := toSliceUnsafe(key) start, stop, step, sliceLen, raised := slice.calcSlice(f, len(s)) if raised != nil { return nil, raised } if step == 1 { return NewUnicodeFromRunes(s[start:stop]).ToObject(), nil } result := make([]rune, 0, sliceLen) for j := start; j < stop; j += step { result = append(result, s[j]) } return NewUnicodeFromRunes([]rune(result)).ToObject(), nil } return nil, f.RaiseType(TypeErrorType, fmt.Sprintf("unicode indices must be integers or slice, not %s", key.typ.Name())) } func unicodeGetNewArgs(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "__getnewargs__", args, UnicodeType); raised != nil { return nil, raised } return NewTuple1(args[0]).ToObject(), nil } func unicodeGT(f *Frame, v, w *Object) (*Object, *BaseException) { return unicodeCompare(f, toUnicodeUnsafe(v), w, False, False, True) } func unicodeHash(f *Frame, o *Object) (*Object, *BaseException) { s := toUnicodeUnsafe(o).Value() l := len(s) if l == 0 { return NewInt(0).ToObject(), nil } h := int(s[0]) << 7 for _, r := range s { h = (1000003 * h) ^ int(r) } h ^= l if h == -1 { h = -2 } return NewInt(h).ToObject(), nil } func unicodeJoin(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkMethodArgs(f, "join", args, UnicodeType, ObjectType); raised != nil { return nil, raised } var result *Object raised := seqApply(f, args[1], func(parts []*Object, _ bool) (raised *BaseException) { result, raised = unicodeJoinParts(f, toUnicodeUnsafe(args[0]), parts) return raised }) if raised != nil { return nil, raised } return result, nil } func unicodeLE(f *Frame, v, w *Object) (*Object, *BaseException) { return unicodeCompare(f, toUnicodeUnsafe(v), w, True, True, False) } func unicodeLen(f *Frame, o *Object) (*Object, *BaseException) { return NewInt(len(toUnicodeUnsafe(o).Value())).ToObject(), nil } func unicodeLT(f *Frame, v, w *Object) (*Object, *BaseException) { return unicodeCompare(f, toUnicodeUnsafe(v), w, True, False, False) } func unicodeMul(f *Frame, v, w *Object) (*Object, *BaseException) { value := toUnicodeUnsafe(v).Value() numChars := len(value) n, ok, raised := strRepeatCount(f, numChars, w) if raised != nil { return nil, raised } if !ok { return NotImplemented, nil } newLen := numChars * n newValue := make([]rune, newLen) for i := 0; i < newLen; i += numChars { copy(newValue[i:], value) } return NewUnicodeFromRunes(newValue).ToObject(), nil } func unicodeNative(f *Frame, o *Object) (reflect.Value, *BaseException) { // Encode to utf-8 when passing data out to Go. s, raised := toUnicodeUnsafe(o).Encode(f, EncodeDefault, EncodeStrict) if raised != nil { return reflect.Value{}, raised } return reflect.ValueOf(s.Value()), nil } func unicodeNE(f *Frame, v, w *Object) (*Object, *BaseException) { return unicodeCompareEq(f, toUnicodeUnsafe(v), w, false) } func unicodeNew(f *Frame, t *Type, args Args, _ KWArgs) (ret *Object, raised *BaseException) { // TODO: Accept keyword arguments: string, encoding, errors. if t != UnicodeType { // Allocate a plain unicode then copy it's value into an object // of the unicode subtype. s, raised := unicodeNew(f, UnicodeType, args, nil) if raised != nil { return nil, raised } result := toUnicodeUnsafe(newObject(t)) result.value = toUnicodeUnsafe(s).Value() return result.ToObject(), nil } expectedTypes := []*Type{ObjectType, StrType, StrType} argc := len(args) if argc < 3 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "__new__", args, expectedTypes...); raised != nil { return nil, raised } if argc == 0 { return NewUnicodeFromRunes(nil).ToObject(), nil } arg0 := args[0] if argc == 1 { if unicode := arg0.typ.slots.Unicode; unicode != nil { ret, raised = unicode.Fn(f, arg0) } else if arg0.typ == UnicodeType { ret = toUnicodeUnsafe(arg0).ToObject() } else if arg0.isInstance(UnicodeType) { // Return a unicode object (not a subtype). ret = NewUnicodeFromRunes(toUnicodeUnsafe(arg0).Value()).ToObject() } else if str := arg0.typ.slots.Str; str != nil { ret, raised = str.Fn(f, arg0) } else { var s *Str if s, raised = Repr(f, arg0); raised == nil { ret = s.ToObject() } } if raised != nil { return nil, raised } u, raised := unicodeCoerce(f, ret) if raised != nil { return nil, raised } return u.ToObject(), nil } if !arg0.isInstance(StrType) { format := "coercing to Unicode: need str, %s found" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, arg0.typ.Name())) } encoding := toStrUnsafe(args[1]).Value() errors := "strict" if argc > 2 { errors = toStrUnsafe(args[2]).Value() } s, raised := toStrUnsafe(arg0).Decode(f, encoding, errors) if raised != nil { return nil, raised } return s.ToObject(), nil } func unicodeRepr(_ *Frame, o *Object) (*Object, *BaseException) { buf := bytes.Buffer{} buf.WriteString("u'") for _, r := range toUnicodeUnsafe(o).Value() { if escape, ok := escapeMap[r]; ok { buf.WriteString(escape) } else if r <= unicode.MaxASCII && unicode.IsPrint(r) { buf.WriteRune(r) } else { buf.Write(escapeRune(r)) } } buf.WriteRune('\'') return NewStr(buf.String()).ToObject(), nil } func unicodeStr(f *Frame, o *Object) (*Object, *BaseException) { ret, raised := toUnicodeUnsafe(o).Encode(f, EncodeDefault, EncodeStrict) if raised != nil { return nil, raised } return ret.ToObject(), nil } func unicodeStrip(f *Frame, args Args, _ KWArgs) (*Object, *BaseException) { expectedTypes := []*Type{UnicodeType, ObjectType} argc := len(args) if argc == 1 { expectedTypes = expectedTypes[:argc] } if raised := checkMethodArgs(f, "strip", args, expectedTypes...); raised != nil { return nil, raised } s := toUnicodeUnsafe(args[0]) charsArg := None if argc > 1 { charsArg = args[1] } matchFunc := unicode.IsSpace if charsArg != None { chars, raised := unicodeCoerce(f, charsArg) if raised != nil { return nil, raised } matchFunc = func(r rune) bool { for _, c := range chars.Value() { if r == c { return true } } return false } } runes := s.Value() numRunes := len(runes) lindex := 0 for ; lindex < numRunes; lindex++ { if !matchFunc(runes[lindex]) { break } } rindex := numRunes for ; rindex > lindex; rindex-- { if !matchFunc(runes[rindex-1]) { break } } result := make([]rune, rindex-lindex) copy(result, runes[lindex:rindex]) return NewUnicodeFromRunes(result).ToObject(), nil } func initUnicodeType(dict map[string]*Object) { dict["__getnewargs__"] = newBuiltinFunction("__getnewargs__", unicodeGetNewArgs).ToObject() dict["encode"] = newBuiltinFunction("encode", unicodeEncode).ToObject() dict["join"] = newBuiltinFunction("join", unicodeJoin).ToObject() dict["strip"] = newBuiltinFunction("strip", unicodeStrip).ToObject() UnicodeType.slots.Add = &binaryOpSlot{unicodeAdd} UnicodeType.slots.Contains = &binaryOpSlot{unicodeContains} UnicodeType.slots.Eq = &binaryOpSlot{unicodeEq} UnicodeType.slots.GE = &binaryOpSlot{unicodeGE} UnicodeType.slots.GetItem = &binaryOpSlot{unicodeGetItem} UnicodeType.slots.GT = &binaryOpSlot{unicodeGT} UnicodeType.slots.Hash = &unaryOpSlot{unicodeHash} UnicodeType.slots.LE = &binaryOpSlot{unicodeLE} UnicodeType.slots.Len = &unaryOpSlot{unicodeLen} UnicodeType.slots.LT = &binaryOpSlot{unicodeLT} UnicodeType.slots.Mul = &binaryOpSlot{unicodeMul} UnicodeType.slots.NE = &binaryOpSlot{unicodeNE} UnicodeType.slots.New = &newSlot{unicodeNew} UnicodeType.slots.Native = &nativeSlot{unicodeNative} UnicodeType.slots.RMul = &binaryOpSlot{unicodeMul} UnicodeType.slots.Repr = &unaryOpSlot{unicodeRepr} UnicodeType.slots.Str = &unaryOpSlot{unicodeStr} } func unicodeCompare(f *Frame, v *Unicode, w *Object, ltResult, eqResult, gtResult *Int) (*Object, *BaseException) { rhs := []rune(nil) if w.isInstance(UnicodeType) { rhs = toUnicodeUnsafe(w).Value() } else if w.isInstance(StrType) { ret, raised := toStrUnsafe(w).Decode(f, EncodeDefault, EncodeStrict) if raised != nil { return nil, raised } rhs = ret.Value() } else { return NotImplemented, nil } switch runeSliceCmp(v.Value(), rhs) { case -1: return ltResult.ToObject(), nil case 0: return eqResult.ToObject(), nil default: return gtResult.ToObject(), nil } } func runeSliceCmp(lhs []rune, rhs []rune) int { lhsLen, rhsLen := len(lhs), len(rhs) minLen := lhsLen if rhsLen < lhsLen { minLen = rhsLen } for i := 0; i < minLen; i++ { if lhs[i] < rhs[i] { return -1 } if lhs[i] > rhs[i] { return 1 } } if lhsLen < rhsLen { return -1 } if lhsLen > rhsLen { return 1 } return 0 } // unicodeCompareEq returns the result of comparing whether v and w are equal // (when eq is true) or unequal (when eq is false). It differs from // unicodeCompare in that it will safely decode w if it has type str and // therefore will not raise UnicodeDecodeError. func unicodeCompareEq(f *Frame, v *Unicode, w *Object, eq bool) (*Object, *BaseException) { if w.isInstance(UnicodeType) { // Do the standard comparison knowing that we won't raise // UnicodeDecodeError for w. return unicodeCompare(f, v, w, GetBool(!eq), GetBool(eq), GetBool(!eq)) } if !w.isInstance(StrType) { return NotImplemented, nil } lhs := v.Value() lhsLen := len(lhs) i := 0 // Decode w as utf-8. for _, r := range toStrUnsafe(w).Value() { // lhs[i] should never be RuneError so the second part of the // condition should catch that case. if i >= lhsLen || lhs[i] != r { return GetBool(!eq).ToObject(), nil } i++ } return GetBool((i == lhsLen) == eq).ToObject(), nil } func unicodeCoerce(f *Frame, o *Object) (*Unicode, *BaseException) { switch { case o.isInstance(StrType): return toStrUnsafe(o).Decode(f, EncodeDefault, EncodeStrict) case o.isInstance(UnicodeType): return toUnicodeUnsafe(o), nil default: format := "coercing to Unicode: need string, %s found" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, o.typ.Name())) } } func unicodeJoinParts(f *Frame, s *Unicode, parts []*Object) (*Object, *BaseException) { numParts := len(parts) if numParts == 0 { return NewUnicode("").ToObject(), nil } sep := s.Value() sepLen := len(sep) unicodeParts := make([]*Unicode, numParts) // Calculate the size of the required buffer. numRunes := (numParts - 1) * len(sep) for i, part := range parts { s, raised := unicodeCoerce(f, part) if raised != nil { return nil, raised } unicodeParts[i] = s numRunes += len(s.Value()) } // Piece together the result string into buf. buf := make([]rune, numRunes) offset := 0 for i, part := range unicodeParts { if i > 0 { copy(buf[offset:offset+sepLen], sep) offset += sepLen } s := part.Value() l := len(s) copy(buf[offset:offset+l], s) offset += l } return NewUnicodeFromRunes(buf).ToObject(), nil } ================================================ FILE: runtime/unicode_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "bytes" "reflect" "testing" "unicode" ) func TestUnicodeNewUnicode(t *testing.T) { cases := []struct { s string want []rune }{ // Invalid utf-8 characters should not be present in unicode // objects, but if that happens they're substituted with the // replacement character U+FFFD. {"foo\xffbar", []rune{'f', 'o', 'o', '\uFFFD', 'b', 'a', 'r'}}, // U+D800 is a surrogate that Python 2.x encodes to UTF-8 as // \xed\xa0\x80 but Go treats each code unit as a bad rune. {"\xed\xa0\x80", []rune{'\uFFFD', '\uFFFD', '\uFFFD'}}, } for _, cas := range cases { got := NewUnicode(cas.s).Value() if !reflect.DeepEqual(got, cas.want) { t.Errorf("NewUnicode(%q) = %v, want %v", cas.s, got, cas.want) } } } func TestUnicodeBinaryOps(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, fn func(f *Frame, v, w *Object) (*Object, *BaseException), v, w *Object) (*Object, *BaseException) { return fn(f, v, w) }) cases := []invokeTestCase{ {args: wrapArgs(Add, NewUnicode("foo"), NewUnicode("bar")), want: NewUnicode("foobar").ToObject()}, {args: wrapArgs(Add, NewUnicode("foo"), "bar"), want: NewUnicode("foobar").ToObject()}, {args: wrapArgs(Add, "foo", NewUnicode("bar")), want: NewUnicode("foobar").ToObject()}, {args: wrapArgs(Add, NewUnicode("baz"), NewUnicode("")), want: NewUnicode("baz").ToObject()}, {args: wrapArgs(Add, NewUnicode(""), newObject(ObjectType)), wantExc: mustCreateException(TypeErrorType, "coercing to Unicode: need string, object found")}, {args: wrapArgs(Add, None, NewUnicode("")), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for +: 'NoneType' and 'unicode'")}, {args: wrapArgs(Mul, NewUnicode(""), 10), want: NewUnicode("").ToObject()}, {args: wrapArgs(Mul, NewUnicode("foo"), -2), want: NewUnicode("").ToObject()}, {args: wrapArgs(Mul, NewUnicode("foobar"), 0), want: NewUnicode("").ToObject()}, {args: wrapArgs(Mul, NewUnicode("aloha"), 2), want: NewUnicode("alohaaloha").ToObject()}, {args: wrapArgs(Mul, 1, NewUnicode("baz")), want: NewUnicode("baz").ToObject()}, {args: wrapArgs(Mul, newObject(ObjectType), NewUnicode("qux")), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'object' and 'unicode'")}, {args: wrapArgs(Mul, NewUnicode("foo"), NewUnicode("")), wantExc: mustCreateException(TypeErrorType, "unsupported operand type(s) for *: 'unicode' and 'unicode'")}, {args: wrapArgs(Mul, NewUnicode("bar"), MaxInt), wantExc: mustCreateException(OverflowErrorType, "result too large")}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestUnicodeCompare(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewUnicode(""), NewUnicode("")), want: compareAllResultEq}, {args: wrapArgs(NewUnicode(""), ""), want: compareAllResultEq}, {args: wrapArgs(NewStr(""), NewUnicode("")), want: compareAllResultEq}, {args: wrapArgs(NewUnicode("樂"), NewUnicode("樂")), want: compareAllResultEq}, {args: wrapArgs(NewUnicode("樂"), "樂"), want: compareAllResultEq}, {args: wrapArgs(NewStr("樂"), NewUnicode("樂")), want: compareAllResultEq}, {args: wrapArgs(NewUnicode("вол"), NewUnicode("волн")), want: compareAllResultLT}, {args: wrapArgs(NewUnicode("вол"), "волн"), want: compareAllResultLT}, {args: wrapArgs(NewStr("вол"), NewUnicode("волн")), want: compareAllResultLT}, {args: wrapArgs(NewUnicode("bar"), NewUnicode("baz")), want: compareAllResultLT}, {args: wrapArgs(NewUnicode("bar"), "baz"), want: compareAllResultLT}, {args: wrapArgs(NewStr("bar"), NewUnicode("baz")), want: compareAllResultLT}, {args: wrapArgs(NewUnicode("abc"), None), want: compareAllResultGT}, } for _, cas := range cases { if err := runInvokeTestCase(compareAll, &cas); err != "" { t.Error(err) } } } func TestUnicodeContains(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewUnicode("foobar"), NewUnicode("foo")), want: True.ToObject()}, {args: wrapArgs(NewUnicode("abcdef"), NewUnicode("bar")), want: False.ToObject()}, {args: wrapArgs(NewUnicode(""), NewUnicode("")), want: True.ToObject()}, {args: wrapArgs(NewUnicode(""), 102.1), wantExc: mustCreateException(TypeErrorType, "coercing to Unicode: need string, float found")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(UnicodeType, "__contains__", &cas); err != "" { t.Error(err) } } } func TestUnicodeEncode(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewUnicode("foo")), want: NewStr("foo").ToObject()}, {args: wrapArgs(NewUnicode("foob\u0300ar"), "utf8"), want: NewStr("foob\u0300ar").ToObject()}, {args: wrapArgs(NewUnicode("foo"), "noexist", "strict"), wantExc: mustCreateException(LookupErrorType, "unknown encoding: noexist")}, {args: wrapArgs(NewUnicodeFromRunes([]rune{'в', 'о', 'л', 'н'}), "utf8", "strict"), want: NewStr("\xd0\xb2\xd0\xbe\xd0\xbb\xd0\xbd").ToObject()}, {args: wrapArgs(NewUnicodeFromRunes([]rune{'\xff'}), "utf8"), want: NewStr("\xc3\xbf").ToObject()}, {args: wrapArgs(NewUnicodeFromRunes([]rune{0xD800})), wantExc: mustCreateException(UnicodeEncodeErrorType, `'utf8' codec can't encode character \ud800 in position 0`)}, {args: wrapArgs(NewUnicodeFromRunes([]rune{unicode.MaxRune + 1}), "utf8", "replace"), want: NewStr("\xef\xbf\xbd").ToObject()}, {args: wrapArgs(NewUnicodeFromRunes([]rune{0xFFFFFF}), "utf8", "ignore"), want: NewStr("").ToObject()}, {args: wrapArgs(NewUnicodeFromRunes([]rune{0xFFFFFF}), "utf8", "noexist"), wantExc: mustCreateException(LookupErrorType, "unknown error handler name 'noexist'")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(UnicodeType, "encode", &cas); err != "" { t.Error(err) } } } func TestUnicodeGetItem(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewUnicode("bar"), 1), want: NewUnicode("a").ToObject()}, {args: wrapArgs(NewUnicode("foo"), 3.14), wantExc: mustCreateException(TypeErrorType, "unicode indices must be integers or slice, not float")}, {args: wrapArgs(NewUnicode("baz"), -1), want: NewUnicode("z").ToObject()}, {args: wrapArgs(NewUnicode("baz"), -4), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(NewUnicode(""), 0), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(NewUnicode("foo"), 3), wantExc: mustCreateException(IndexErrorType, "index out of range")}, {args: wrapArgs(NewUnicode("bar"), newTestSlice(None, 2)), want: NewStr("ba").ToObject()}, {args: wrapArgs(NewUnicode("bar"), newTestSlice(1, 3)), want: NewStr("ar").ToObject()}, {args: wrapArgs(NewUnicode("bar"), newTestSlice(1, None)), want: NewStr("ar").ToObject()}, {args: wrapArgs(NewUnicode("foobarbaz"), newTestSlice(1, 8, 2)), want: NewStr("obra").ToObject()}, {args: wrapArgs(NewUnicode("bar"), newTestSlice(1, 2, 0)), wantExc: mustCreateException(ValueErrorType, "slice step cannot be zero")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(UnicodeType, "__getitem__", &cas); err != "" { t.Error(err) } } } func TestUnicodeHash(t *testing.T) { truncateInt := func(i int64) int { return int(i) } // Support for 32bit systems. cases := []invokeTestCase{ {args: wrapArgs(NewUnicode("foo")), want: NewInt(truncateInt(-4177197833195190597)).ToObject()}, {args: wrapArgs(NewUnicode("bar")), want: NewInt(truncateInt(327024216814240868)).ToObject()}, {args: wrapArgs(NewUnicode("baz")), want: NewInt(truncateInt(327024216814240876)).ToObject()}, {args: wrapArgs(NewUnicode("")), want: NewInt(0).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(UnicodeType, "__hash__", &cas); err != "" { t.Error(err) } } } func TestUnicodeLen(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewUnicode("foo")), want: NewInt(3).ToObject()}, {args: wrapArgs(NewUnicode("")), want: NewInt(0).ToObject()}, {args: wrapArgs(NewUnicode("волн")), want: NewInt(4).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(UnicodeType, "__len__", &cas); err != "" { t.Error(err) } } } func TestUnicodeMethods(t *testing.T) { cases := []struct { methodName string args Args want *Object wantExc *BaseException }{ {"join", wrapArgs(NewUnicode(","), newTestList("foo", "bar")), NewUnicode("foo,bar").ToObject(), nil}, {"join", wrapArgs(NewUnicode(":"), newTestList(NewUnicode("foo"), "bar", NewUnicode("baz"))), NewUnicode("foo:bar:baz").ToObject(), nil}, {"join", wrapArgs(NewUnicode("nope"), NewTuple()), NewUnicode("").ToObject(), nil}, {"join", wrapArgs(NewUnicode("nope"), newTestTuple(NewUnicode("foo"))), NewUnicode("foo").ToObject(), nil}, {"join", wrapArgs(NewUnicode(","), newTestList("foo", "bar", 3.14)), nil, mustCreateException(TypeErrorType, "coercing to Unicode: need string, float found")}, {"strip", wrapArgs(NewUnicode("foo ")), NewStr("foo").ToObject(), nil}, {"strip", wrapArgs(NewUnicode(" foo bar ")), NewStr("foo bar").ToObject(), nil}, {"strip", wrapArgs(NewUnicode("foo foo"), "o"), NewStr("foo f").ToObject(), nil}, {"strip", wrapArgs(NewUnicode("foo bar"), "abr"), NewStr("foo ").ToObject(), nil}, {"strip", wrapArgs(NewUnicode("foo"), NewUnicode("o")), NewUnicode("f").ToObject(), nil}, {"strip", wrapArgs(NewUnicode("123"), 3), nil, mustCreateException(TypeErrorType, "coercing to Unicode: need string, int found")}, {"strip", wrapArgs(NewUnicode("foo"), "bar", "baz"), nil, mustCreateException(TypeErrorType, "'strip' of 'unicode' requires 2 arguments")}, {"strip", wrapArgs(NewUnicode("foo"), NewUnicode("o")), NewUnicode("f").ToObject(), nil}, } for _, cas := range cases { testCase := invokeTestCase{args: cas.args, want: cas.want, wantExc: cas.wantExc} if err := runInvokeMethodTestCase(UnicodeType, cas.methodName, &testCase); err != "" { t.Error(err) } } } func TestUnicodeNative(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, s *Unicode) (string, *BaseException) { native, raised := ToNative(f, s.ToObject()) if raised != nil { return "", raised } got, ok := native.Interface().(string) if raised := Assert(f, GetBool(ok).ToObject(), nil); raised != nil { return "", raised } return got, nil }) cases := []invokeTestCase{ {args: wrapArgs(NewUnicode("волн")), want: NewStr("волн").ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(fun, &cas); err != "" { t.Error(err) } } } func TestUnicodeRepr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewUnicode("foo")), want: NewStr("u'foo'").ToObject()}, {args: wrapArgs(NewUnicode("on\nmultiple\nlines")), want: NewStr(`u'on\nmultiple\nlines'`).ToObject()}, {args: wrapArgs(NewUnicode("a\u0300")), want: NewStr(`u'a\u0300'`).ToObject()}, {args: wrapArgs(NewUnicodeFromRunes([]rune{'h', 'o', 'l', 0xFF})), want: NewStr(`u'hol\xff'`).ToObject()}, {args: wrapArgs(NewUnicodeFromRunes([]rune{0x10163})), want: NewStr(`u'\U00010163'`).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(UnicodeType, "__repr__", &cas); err != "" { t.Error(err) } } } func TestUnicodeNew(t *testing.T) { fooType := newTestClass("Foo", []*Type{ObjectType}, newStringDict(map[string]*Object{ "__unicode__": newBuiltinFunction("__unicode__", func(f *Frame, args Args, kwargs KWArgs) (*Object, *BaseException) { return NewStr("foo").ToObject(), nil }).ToObject(), })) strictEqType := newTestClassStrictEq("StrictEq", UnicodeType) cases := []invokeTestCase{ {args: wrapArgs(UnicodeType), want: NewUnicode("").ToObject()}, {args: wrapArgs(UnicodeType, NewUnicode("foo")), want: NewUnicode("foo").ToObject()}, {args: wrapArgs(UnicodeType, newObject(fooType)), want: NewUnicode("foo").ToObject()}, {args: wrapArgs(UnicodeType, "foobar"), want: NewUnicode("foobar").ToObject()}, {args: wrapArgs(UnicodeType, "foo\xffbar"), wantExc: mustCreateException(UnicodeDecodeErrorType, "'utf8' codec can't decode byte 0xff in position 3")}, {args: wrapArgs(UnicodeType, 123), want: NewUnicode("123").ToObject()}, {args: wrapArgs(UnicodeType, 3.14, "utf8"), wantExc: mustCreateException(TypeErrorType, "coercing to Unicode: need str, float found")}, {args: wrapArgs(UnicodeType, "baz", "utf8"), want: NewUnicode("baz").ToObject()}, {args: wrapArgs(UnicodeType, "baz", "utf-8"), want: NewUnicode("baz").ToObject()}, {args: wrapArgs(UnicodeType, "foo\xffbar", "utf_8"), wantExc: mustCreateException(UnicodeDecodeErrorType, "'utf_8' codec can't decode byte 0xff in position 3")}, {args: wrapArgs(UnicodeType, "foo\xffbar", "UTF8", "ignore"), want: NewUnicode("foobar").ToObject()}, {args: wrapArgs(UnicodeType, "foo\xffbar", "utf8", "replace"), want: NewUnicode("foo\ufffdbar").ToObject()}, {args: wrapArgs(UnicodeType, "\xff", "utf-8", "noexist"), wantExc: mustCreateException(LookupErrorType, "unknown error handler name 'noexist'")}, {args: wrapArgs(UnicodeType, "\xff", "utf16"), wantExc: mustCreateException(LookupErrorType, "unknown encoding: utf16")}, {args: wrapArgs(strictEqType, NewUnicode("foo")), want: (&Unicode{Object{typ: strictEqType}, bytes.Runes([]byte("foo"))}).ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(UnicodeType, "__new__", &cas); err != "" { t.Error(err) } } } func TestUnicodeNewNotSubtype(t *testing.T) { cas := invokeTestCase{args: wrapArgs(IntType), wantExc: mustCreateException(TypeErrorType, "unicode.__new__(int): int is not a subtype of unicode")} if err := runInvokeMethodTestCase(UnicodeType, "__new__", &cas); err != "" { t.Error(err) } } func TestUnicodeNewSubclass(t *testing.T) { fooType := newTestClass("Foo", []*Type{UnicodeType}, NewDict()) bar := (&Unicode{Object{typ: fooType}, bytes.Runes([]byte("bar"))}).ToObject() fun := wrapFuncForTest(func(f *Frame) *BaseException { got, raised := UnicodeType.Call(f, []*Object{bar}, nil) if raised != nil { return raised } if got.typ != UnicodeType { t.Errorf(`unicode(Foo("bar")) = %v, want u"bar"`, got) return nil } ne, raised := NE(f, got, NewUnicode("bar").ToObject()) if raised != nil { return raised } isTrue, raised := IsTrue(f, ne) if raised != nil { return raised } if isTrue { t.Errorf(`unicode(Foo("bar")) = %v, want u"bar"`, got) } return nil }) if err := runInvokeTestCase(fun, &invokeTestCase{want: None}); err != "" { t.Error(err) } } func TestUnicodeStr(t *testing.T) { cases := []invokeTestCase{ {args: wrapArgs(NewUnicode("foo")), want: NewStr("foo").ToObject()}, {args: wrapArgs(NewUnicode("on\nmultiple\nlines")), want: NewStr("on\nmultiple\nlines").ToObject()}, {args: wrapArgs(NewUnicode("a\u0300")), want: NewStr("a\u0300").ToObject()}, } for _, cas := range cases { if err := runInvokeMethodTestCase(UnicodeType, "__str__", &cas); err != "" { t.Error(err) } } } ================================================ FILE: runtime/weakref.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "reflect" "runtime" "sync" "sync/atomic" "unsafe" ) var ( // WeakRefType is the object representing the Python 'weakref' type. WeakRefType = newBasisType("weakref", reflect.TypeOf(WeakRef{}), toWeakRefUnsafe, ObjectType) ) type weakRefState int const ( weakRefStateNew weakRefState = iota weakRefStateUsed weakRefStateDead ) // WeakRef represents Python 'weakref' objects. type WeakRef struct { Object ptr uintptr mutex sync.Mutex state weakRefState callbacks []*Object hash *Object } func toWeakRefUnsafe(o *Object) *WeakRef { return (*WeakRef)(o.toPointer()) } // get returns r's referent, or nil if r is "dead". func (r *WeakRef) get() *Object { if r.state == weakRefStateDead { return nil } r.state = weakRefStateUsed return (*Object)(unsafe.Pointer(r.ptr)) } // ToObject upcasts r to an Object. func (r *WeakRef) ToObject() *Object { return &r.Object } func weakRefCall(f *Frame, callable *Object, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionArgs(f, "__call__", args); raised != nil { return nil, raised } r := toWeakRefUnsafe(callable) r.mutex.Lock() o := r.get() r.mutex.Unlock() if o == nil { o = None } return o, nil } func weakRefHash(f *Frame, o *Object) (result *Object, raised *BaseException) { r := toWeakRefUnsafe(o) var referent *Object r.mutex.Lock() if r.hash != nil { result = r.hash } else { referent = r.get() } r.mutex.Unlock() if referent != nil { var hash *Int hash, raised = Hash(f, referent) if raised == nil { result = hash.ToObject() r.mutex.Lock() r.hash = result r.mutex.Unlock() } } else if result == nil { raised = f.RaiseType(TypeErrorType, "weak object has gone away") } return result, raised } func weakRefNew(f *Frame, t *Type, args Args, _ KWArgs) (*Object, *BaseException) { if raised := checkFunctionVarArgs(f, "__new__", args, ObjectType); raised != nil { return nil, raised } argc := len(args) if argc > 2 { format := "__new__ expected at most 2 arguments, got %d" return nil, f.RaiseType(TypeErrorType, fmt.Sprintf(format, argc)) } o := args[0] nilPtr := unsafe.Pointer(nil) addr := (*unsafe.Pointer)(unsafe.Pointer(&o.ref)) var r *WeakRef // Atomically fetch or initialize o.ref. for { p := atomic.LoadPointer(addr) if p != nilPtr { r = (*WeakRef)(p) break } else { r = &WeakRef{Object: Object{typ: WeakRefType}, ptr: uintptr(o.toPointer())} if atomic.CompareAndSwapPointer(addr, nilPtr, r.toPointer()) { runtime.SetFinalizer(o, weakRefFinalizeReferent) break } } } if argc > 1 { r.mutex.Lock() r.callbacks = append(r.callbacks, args[1]) r.mutex.Unlock() } return r.ToObject(), nil } func weakRefRepr(f *Frame, o *Object) (*Object, *BaseException) { r := toWeakRefUnsafe(o) r.mutex.Lock() p := r.get() r.mutex.Unlock() s := "dead" if p != nil { s = fmt.Sprintf("to '%s' at %p", p.Type().Name(), p) } return NewStr(fmt.Sprintf("", r, s)).ToObject(), nil } func initWeakRefType(map[string]*Object) { WeakRefType.slots.Call = &callSlot{weakRefCall} WeakRefType.slots.Hash = &unaryOpSlot{weakRefHash} WeakRefType.slots.New = &newSlot{weakRefNew} WeakRefType.slots.Repr = &unaryOpSlot{weakRefRepr} } func weakRefFinalizeReferent(o *Object) { // Note that although o should be the last reference to that object // (since this is its finalizer), in the time between the runtime // scheduling this finalizer and the Lock() call below, r may have // handed out another reference to o. So we can't simply mark r "dead". addr := (*unsafe.Pointer)(unsafe.Pointer(&o.ref)) r := (*WeakRef)(atomic.LoadPointer(addr)) numCallbacks := 0 var callbacks []*Object r.mutex.Lock() switch r.state { case weakRefStateNew: // State "new" means that no references have been handed out by // r and therefore o is the only live reference. r.state = weakRefStateDead numCallbacks = len(r.callbacks) callbacks = make([]*Object, numCallbacks) copy(callbacks, r.callbacks) case weakRefStateUsed: // Most likely it's safe to mark r "dead" at this point, but // because a reference was handed out at some point, play it // safe and reset the finalizer. If no more references are // handed out before the next finalize then it will be "dead". r.state = weakRefStateNew runtime.SetFinalizer(o, weakRefFinalizeReferent) } r.mutex.Unlock() // Don't hold r.mutex while invoking callbacks in case they access r // and attempt to acquire the mutex. for i := numCallbacks - 1; i >= 0; i-- { f := NewRootFrame() if _, raised := callbacks[i].Call(f, Args{r.ToObject()}, nil); raised != nil { Stderr.writeString(FormatExc(f)) } } } ================================================ FILE: runtime/weakref_test.go ================================================ // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grumpy import ( "fmt" "runtime" "testing" "time" ) func TestWeakRefCall(t *testing.T) { aliveRef, alive, deadRef := makeWeakRefsForTest() dupRef := newTestWeakRef(alive, nil) cases := []invokeTestCase{ {args: wrapArgs(aliveRef), want: alive}, {args: wrapArgs(dupRef), want: alive}, {args: wrapArgs(deadRef), want: None}, {args: wrapArgs(aliveRef, 123), wantExc: mustCreateException(TypeErrorType, "'__call__' requires 0 arguments")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(WeakRefType, "__call__", &cas); err != "" { t.Error(err) } } runtime.KeepAlive(alive) } func TestWeakRefHash(t *testing.T) { aliveRef, alive, deadRef := makeWeakRefsForTest() hashedRef, hashed, _ := makeWeakRefsForTest() if _, raised := Hash(NewRootFrame(), hashedRef.ToObject()); raised != nil { t.Fatal(raised) } runtime.KeepAlive(hashed) hashed = nil weakRefMustDie(hashedRef) unhashable := NewList().ToObject() unhashableRef := newTestWeakRef(unhashable, nil) cases := []invokeTestCase{ {args: wrapArgs(aliveRef), want: NewInt(hashString("foo")).ToObject()}, {args: wrapArgs(deadRef), wantExc: mustCreateException(TypeErrorType, "weak object has gone away")}, {args: wrapArgs(hashedRef), want: NewInt(hashString("foo")).ToObject()}, {args: wrapArgs(unhashableRef), wantExc: mustCreateException(TypeErrorType, "unhashable type: 'list'")}, } for _, cas := range cases { if err := runInvokeMethodTestCase(WeakRefType, "__hash__", &cas); err != "" { t.Error(err) } } runtime.KeepAlive(alive) runtime.KeepAlive(unhashable) } func TestWeakRefNew(t *testing.T) { alive := NewStr("foo").ToObject() aliveRef := newTestWeakRef(alive, nil) cases := []invokeTestCase{ {args: wrapArgs(alive), want: aliveRef.ToObject()}, {wantExc: mustCreateException(TypeErrorType, "'__new__' requires 1 arguments")}, {args: wrapArgs("foo", "bar", "baz"), wantExc: mustCreateException(TypeErrorType, "__new__ expected at most 2 arguments, got 3")}, } for _, cas := range cases { if err := runInvokeTestCase(WeakRefType.ToObject(), &cas); err != "" { t.Error(err) } } runtime.KeepAlive(alive) } func TestWeakRefNewCallback(t *testing.T) { callbackChannel := make(chan *WeakRef) callback := wrapFuncForTest(func(f *Frame, r *WeakRef) { callbackChannel <- r }) r := newTestWeakRef(newObject(ObjectType), callback) weakRefMustDie(r) if r.get() != nil { t.Fatalf("expected weakref %v to be dead", r) } if callbackGot := <-callbackChannel; callbackGot != r { t.Fatalf("callback got %v, want %v", callbackGot, r) } } func TestWeakRefNewCallbackRaises(t *testing.T) { // It's not easy to verify that the exception is output properly, but // we can at least make sure the program doesn't blow up if the // callback raises. callback := wrapFuncForTest(func(f *Frame, r *WeakRef) *BaseException { return f.RaiseType(RuntimeErrorType, "foo") }) r := newTestWeakRef(newObject(ObjectType), callback) weakRefMustDie(r) if r.get() != nil { t.Fatalf("expected weakref %v to be dead", r) } } func TestWeakRefStrRepr(t *testing.T) { aliveRef, alive, deadRef := makeWeakRefsForTest() cases := []invokeTestCase{ {args: wrapArgs(aliveRef), want: NewStr(fmt.Sprintf("", aliveRef, alive)).ToObject()}, {args: wrapArgs(deadRef), want: NewStr(fmt.Sprintf("", deadRef)).ToObject()}, } for _, cas := range cases { if err := runInvokeTestCase(wrapFuncForTest(ToStr), &cas); err != "" { t.Error(err) } if err := runInvokeTestCase(wrapFuncForTest(Repr), &cas); err != "" { t.Error(err) } } runtime.KeepAlive(alive) } func newTestWeakRef(o, callback *Object) *WeakRef { args := Args{o} if callback != nil { args = Args{o, callback} } return toWeakRefUnsafe(mustNotRaise(WeakRefType.Call(NewRootFrame(), args, nil))) } func makeWeakRefsForTest() (*WeakRef, *Object, *WeakRef) { alive := NewStr("foo").ToObject() aliveRef := newTestWeakRef(alive, nil) dead := NewFloat(3.14).ToObject() deadRef := newTestWeakRef(dead, nil) dead = nil weakRefMustDie(deadRef) return aliveRef, alive, deadRef } func weakRefMustDie(r *WeakRef) { r.mutex.Lock() o := r.get() r.mutex.Unlock() if o == nil { return } doneChannel := make(chan bool) callback := wrapFuncForTest(func(f *Frame, r *WeakRef) { close(doneChannel) }) mustNotRaise(WeakRefType.Call(NewRootFrame(), Args{o, callback}, nil)) o = nil timeoutChannel := make(chan bool) go func() { // Finalizers run some time after GC, thus the Sleep call. In // our case o's finalizer will have to run twice, so loop and // GC repeatedly. In theory, twice should be enough, but in // practice there are race conditions and things to contend // with so just loop a bunch of times. wait := 10 * time.Millisecond for t := time.Duration(0); t < time.Second; t += wait { runtime.GC() time.Sleep(wait) } close(timeoutChannel) }() select { case <-doneChannel: return case <-timeoutChannel: panic(fmt.Sprintf("weakref %v did not die", r)) } } ================================================ FILE: testing/assert_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=g-equals-none assert object() assert True assert not False assert not None ================================================ FILE: testing/assign_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=unbalanced-tuple-unpacking class Foo(object): pass foo = 1 assert foo == 1 foo, bar = 2, 3 assert foo == 2 assert bar == 3 (foo, bar), baz = (4, 5), 6 assert foo == 4 assert bar == 5 assert baz == 6 foo = [7, 8, 9] bar = foo assert bar == [7, 8, 9] try: bar, baz = foo except ValueError as e: assert str(e) == 'too many values to unpack' else: raise AssertionError('this was supposed to raise an exception') try: bar, baz, qux, quux = foo except ValueError as e: assert str(e) == 'need more than 3 values to unpack' else: raise AssertionError('this was supposed to raise an exception') foo = Foo() foo.bar = 1 assert foo.bar == 1 foo.bar, baz = 2, 3 assert foo.bar == 2 assert baz == 3 foo.bar, (foo.baz, qux) = 4, (5, 6) assert foo.bar == 4 assert foo.baz == 5 assert qux == 6 foo = bar = baz = 7 assert foo == 7 assert bar == 7 assert baz == 7 foo, bar = baz = 8, 9 assert foo == 8 assert bar == 9 assert baz == (8, 9) foo = 1 foo += 3 assert foo == 4 foo /= 2 assert foo == 2 foo *= 6 assert foo == 12 foo %= 5 assert foo == 2 foo -= 3 assert foo == -1 foo = [] bar = foo foo += ["bar", "baz"] assert foo == ["bar", "baz"] foo *= 2 assert foo == ["bar", "baz", "bar", "baz"] assert bar is foo # Multiple target assignment should only evaluate rhs once. def foo(): # pylint: disable=function-redefined foo_ran[0] += 1 return 'bar' foo_ran = [0] baz = qux = foo() assert baz == 'bar' assert qux == 'bar' assert foo_ran == [1] ================================================ FILE: testing/builtin_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=g-equals-none # abs(x) assert abs(1) == 1 assert abs(-1) == 1 assert isinstance(abs(-1), int) assert abs(long(2)) == 2 assert abs(long(-2)) == 2 assert isinstance(abs(long(-2)), long) assert abs(3.4) == 3.4 assert abs(-3.4) == 3.4 assert isinstance(abs(-3.4), float) assert abs(complex(0, 0)) == 0.0 assert abs(complex(3, 4)) == 5.0 assert abs(-complex(3, 4)) == 5.0 assert abs(complex(0.123456e-3, 0)) == 0.000123456 assert abs(complex(0.123456e-3, 3.14151692e+7)) == 31415169.2 assert isinstance(abs(complex(3, 4)), float) assert repr(abs(complex(-float('inf'), 1.2))) == 'inf' assert repr(abs(complex(float('nan'), float('inf')))) == 'inf' assert repr(abs(complex(3.14, float('nan')))) == 'nan' try: abs('a') except TypeError as e: assert str(e) == "bad operand type for abs(): 'str'" else: raise AssertionError('this was supposed to raise an exception') # all(iterable) assert all([1, 2, 3]) assert all([]) assert not all([1, 1, 1, 0, 1]) assert all([True, True]) assert not all([False, True, True]) assert all('') assert all('abc') try: all(13) except TypeError as e: assert str(e) == "'int' object is not iterable" else: raise AssertionError('this was supposed to raise an exception') # any(iterable) assert any([1, 2, 3]) assert not any([]) assert any([1, 1, 1, 0, 1]) assert not any([0, 0, 0]) assert any([True, True]) assert any([False, True, True]) assert not any([False, False, False]) assert not any('') assert any('abc') try: any(13) except TypeError as e: assert str(e) == "'int' object is not iterable" else: raise AssertionError('this was supposed to raise an exception') # callable(x) assert not callable(1) assert not callable(0.1) assert not callable([1, 2, 3]) assert not callable((1, 2, 3)) assert not callable({'foo': 1, 'bar': 2}) assert callable(lambda x: x + 1) def foo(x): pass assert callable(foo) class bar(object): def __call__(self, *args, **kwargs): pass assert callable(bar) assert callable(bar()) # cmp(x) # Test simple cases. assert cmp(1, 2) == -1 assert cmp(3, 3) == 0 assert cmp(5, 4) == 1 class Lt(object): def __init__(self, x): self.lt_called = False self.x = x def __lt__(self, other): self.lt_called = True return self.x < other.x class Eq(object): def __init__(self, x): self.eq_called = False self.x = x def __eq__(self, other): self.eq_called = True return self.x == other.x class Gt(object): def __init__(self, x): self.gt_called = False self.x = x def __gt__(self, other): self.gt_called = True return self.x > other.x class RichCmp(Lt, Eq, Gt): def __init__(self, x): self.x = x class Cmp(object): def __init__(self, x): self.cmp_called = False self.x = x def __cmp__(self, other): self.cmp_called = True if self.x < other.x: return -1 elif self.x > other.x: return 1 else: return 0 class NoCmp(object): def __init__(self, x): self.x = x # Test 3-way compare in terms of rich compare. a, b = RichCmp(1), RichCmp(2) assert cmp(a, b) == -1 assert a.lt_called a, b = RichCmp(3), RichCmp(3) assert cmp(a, b) == 0 assert a.eq_called a, b = RichCmp(5), RichCmp(4) assert cmp(a, b) == 1 assert a.gt_called # Test pure 3-way compare. a, b = Cmp(1), Cmp(2) assert cmp(a, b) == -1 assert a.cmp_called a, b = Cmp(3), Cmp(3) assert cmp(a, b) == 0 assert a.cmp_called a, b = Cmp(5), Cmp(4) assert cmp(a, b) == 1 # Test mixed 3-way and rich compare. a, b = RichCmp(1), Cmp(2) assert cmp(a, b) == -1 assert a.lt_called assert not b.cmp_called a, b = Cmp(1), RichCmp(2) assert cmp(a, b) == -1 assert not a.cmp_called assert b.gt_called a, b = RichCmp(3), Cmp(3) assert cmp(a, b) == 0 assert a.eq_called assert not b.cmp_called a, b = Cmp(3), RichCmp(3) assert cmp(a, b) == 0 assert not a.cmp_called assert b.eq_called a, b = RichCmp(5), Cmp(4) assert cmp(a, b) == 1 assert a.gt_called assert not b.cmp_called a, b = Cmp(5), RichCmp(4) assert cmp(a, b) == 1 assert not a.cmp_called assert b.gt_called # Test compare on only one object. a, b = Cmp(1), NoCmp(2) assert cmp(a, b) == -1 assert a.cmp_called a, b = NoCmp(1), Cmp(2) assert cmp(a, b) == -1 assert b.cmp_called # Test delattr class Foo(object): pass setattr(Foo, "a", 1) assert Foo.a == 1 # pylint: disable=no-member delattr(Foo, "a") assert getattr(Foo, "a", None) is None try: delattr(Foo, 1, "a") assert AssertionError except TypeError: pass try: delattr(Foo) assert AssertionError except TypeError: pass try: delattr(Foo, "a", 1) assert AssertionError except TypeError: pass # Test setattr setattr(Foo, "a", 1) assert Foo.a == 1 # pylint: disable=no-member try: setattr(Foo, 1, "a") assert AssertionError except TypeError: pass try: setattr(Foo) assert AssertionError except TypeError: pass # Test sorted assert sorted([3, 2, 4, 1]) == [1, 2, 3, 4] assert sorted([]) == [] assert sorted(["a", "e", "c", "b"]) == ["a", "b", "c", "e"] assert sorted((3, 1, 5, 2, 4)) == [1, 2, 3, 4, 5] assert sorted({"foo": 1, "bar": 2}) == ["bar", "foo"] # Test zip assert zip('abc', (0, 1, 2)) == [('a', 0), ('b', 1), ('c', 2)] assert list(zip('abc', range(6))) == zip('abc', range(6)) assert list(zip('abcdef', range(3))) == zip('abcdef', range(3)) assert list(zip('abcdef')) == zip('abcdef') assert list(zip()) == zip() assert [tuple(list(pair)) for pair in zip('abc', 'def')] == zip('abc', 'def') assert [pair for pair in zip('abc', 'def')] == zip('abc', 'def') assert zip({'b': 1, 'a': 2}) == [('a',), ('b',)] assert zip(range(5)) == [(0,), (1,), (2,), (3,), (4,)] assert zip(xrange(5)) == [(0,), (1,), (2,), (3,), (4,)] assert zip([1, 2, 3], [1], [4, 5, 6]) == [(1, 1, 4)] assert zip([1], [1, 2, 3], [4, 5, 6]) == [(1, 1, 4)] assert zip([4, 5, 6], [1], [1, 2, 3]) == [(4, 1, 1)] assert zip([1], [1, 2, 3], [4]) == [(1, 1, 4)] assert zip([1, 2], [1, 2, 3], [4]) == [(1, 1, 4)] assert zip([1, 2, 3, 4], [1, 2, 3], [4]) == [(1, 1, 4)] assert zip([1], [1, 2], [4, 2, 4]) == [(1, 1, 4)] assert zip([1, 2, 3], [1, 2], [4]) == [(1, 1, 4)] assert zip([1, 2, 3], [1, 2], [4], []) == [] assert zip([], [1], [1, 2], [1, 2, 3]) == [] try: zip([1, 2, 3], [1, 2], [4], None) raise AssertionError except TypeError: pass # Test map assert map(str, []) == [] assert map(str, [1, 2, 3]) == ["1", "2", "3"] assert map(str, (1, 2, 3)) == ["1", "2", "3"] # assert map(str, (1.0, 2.0, 3.0)) == ["1", "2", "3"] assert map(str, range(3)) == ["0", "1", "2"] assert map(str, xrange(3)) == ["0", "1", "2"] assert map(int, ["1", "2", "3"]) == [1, 2, 3] assert map(int, "123") == [1, 2, 3] assert map(int, {"1": "a", "2": "b"}) == [1, 2] assert map(int, {1: "a", 2: "b"}) == [1, 2] assert map(lambda a, b: (str(a), float(b or 0) + 0.1), [1, 2, 3], [1, 2]) == [('1', 1.1), ('2', 2.1), ('3', 0.1)] assert map(None, [1, 2, 3]) == [1, 2, 3] a = [1, 2, 3] assert map(None, a) == a assert map(None, a) is not a assert map(None, (1, 2, 3)) == [1, 2, 3] # divmod(v, w) import sys assert divmod(12, 7) == (1, 5) assert divmod(-12, 7) == (-2, 2) assert divmod(12, -7) == (-2, -2) assert divmod(-12, -7) == (1, -5) assert divmod(-sys.maxsize - 1, -1) == (sys.maxsize + 1, 0) assert isinstance(divmod(12, 7), tuple) assert isinstance(divmod(12, 7)[0], int) assert isinstance(divmod(12, 7)[1], int) assert divmod(long(7), long(3)) == (2L, 1L) assert divmod(long(3), long(-7)) == (-1L, -4L) assert divmod(long(sys.maxsize), long(-sys.maxsize)) == (-1L, 0L) assert divmod(long(-sys.maxsize), long(1)) == (-sys.maxsize, 0L) assert divmod(long(-sys.maxsize), long(-1)) == (sys.maxsize, 0L) assert isinstance(divmod(long(7), long(3)), tuple) assert isinstance(divmod(long(7), long(3))[0], long) assert isinstance(divmod(long(7), long(3))[1], long) assert divmod(3.25, 1.0) == (3.0, 0.25) assert divmod(-3.25, 1.0) == (-4.0, 0.75) assert divmod(3.25, -1.0) == (-4.0, -0.75) assert divmod(-3.25, -1.0) == (3.0, -0.25) assert isinstance(divmod(3.25, 1.0), tuple) assert isinstance(divmod(3.25, 1.0)[0], float) assert isinstance(divmod(3.25, 1.0)[1], float) try: divmod('a', 'b') except TypeError as e: assert str(e) == "unsupported operand type(s) for divmod(): 'str' and 'str'" else: assert AssertionError # Check for a bug where zip() and map() were not properly cleaning their # internal exception state. See: # https://github.com/google/grumpy/issues/305 sys.exc_clear() zip((1, 3), (2, 4)) assert not any(sys.exc_info()) map(int, (1, 2, 3)) assert not any(sys.exc_info()) ================================================ FILE: testing/class_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. class Foo(object): a = 3 assert a == 3 def bar(self): assert isinstance(self, Foo) return 'bar' baz = bar assert Foo.a == 3 Foo.a = 4 assert Foo.a == 4 foo = Foo() assert isinstance(foo, Foo) assert foo.a == 4 foo.a = 5 assert foo.a == 5 assert Foo.a == 4 assert foo.bar() == 'bar' assert foo.baz() == 'bar' foo.b = 10 del foo.b assert not hasattr(foo, 'b') try: del foo.b except AttributeError: pass else: raise AssertionError ================================================ FILE: testing/compare_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. assert 1 < 100 assert -10 <= "foo" assert "bar" <= "bar" assert (1, "a", 3) == (1, "a", 3) assert 15 != 16 assert [] != None # pylint: disable=g-equals-none,g-explicit-bool-comparison assert int >= "az" assert "foo" >= "foo" assert True > False # Test rich comparisons. class RichCmp(object): def __init__(self, x): self.x = x self.lt_called = False self.le_called = False self.eq_called = False self.ge_called = False self.gt_called = False def __lt__(self, other): self.lt_called = True return self.x < other.x def __le__(self, other): self.le_called = True return self.x <= other.x def __eq__(self, other): self.eq_called = True return self.x == other.x def __ge__(self, other): self.ge_called = True return self.x >= other.x def __gt__(self, other): self.gt_called = True return self.x > other.x class Cmp(object): def __init__(self, x): self.cmp_called = False self.x = x def __cmp__(self, other): self.cmp_called = True return cmp(self.x, other.x) # Test that rich comparison methods are called. a, b = RichCmp(1), RichCmp(2) assert a < b assert a.lt_called a, b = RichCmp(1), RichCmp(2) assert a <= b assert a.le_called a, b = RichCmp(3), RichCmp(3) assert a == b assert a.eq_called a, b = RichCmp(5), RichCmp(4) assert a >= b assert a.ge_called a, b = RichCmp(5), RichCmp(4) assert a > b assert a.gt_called # Test rich comparison falling back to a 3-way comparison a, b = Cmp(1), Cmp(2) assert a < b assert a.cmp_called a, b = Cmp(1), Cmp(2) assert a <= b assert a.cmp_called a, b = Cmp(3), Cmp(3) assert a == b assert a.cmp_called a, b = Cmp(5), Cmp(4) assert a > b assert a.cmp_called a, b = Cmp(5), Cmp(4) assert a >= b assert a.cmp_called ================================================ FILE: testing/complex_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. assert repr(1j) == "1j" assert repr(complex()) == "0j" assert repr(complex('nan-nanj')) == '(nan+nanj)' assert repr(complex('-Nan+NaNj')) == '(nan+nanj)' assert repr(complex('inf-infj')) == '(inf-infj)' assert repr(complex('+inf+infj')) == '(inf+infj)' assert repr(complex('-infINIty+infinityj')) == '(-inf+infj)' assert complex(1.8456e3) == (1845.6+0j) assert complex('1.8456e3') == (1845.6+0j) assert complex(0, -365.12) == -365.12j assert complex('-365.12j') == -365.12j assert complex(-1.23E2, -45.678e1) == (-123-456.78j) assert complex('-1.23e2-45.678e1j') == (-123-456.78j) assert complex(21.98, -1) == (21.98-1j) assert complex('21.98-j') == (21.98-1j) assert complex('-j') == -1j assert complex('+j') == 1j assert complex('j') == 1j assert complex(' \t \n \r ( \t \n \r 2.1-3.4j \t \n \r ) \t \n \r ') == (2.1-3.4j) assert complex(complex(complex(3.14))) == (3.14+0j) assert complex(complex(1, -2), .151692) == (1-1.848308j) assert complex(complex(3.14), complex(-0.151692)) == (3.14-0.151692j) assert complex(complex(-1, 2), complex(3, -4)) == (3+5j) try: complex('((2.1-3.4j))') except ValueError as e: assert str(e) == "complex() arg is a malformed string" else: raise AssertionError('this was supposed to raise an exception') try: complex('3.14 - 15.16 j') except ValueError as e: assert str(e) == "complex() arg is a malformed string" else: raise AssertionError('this was supposed to raise an exception') try: complex('foo') except ValueError as e: assert str(e) == "complex() arg is a malformed string" else: raise AssertionError('this was supposed to raise an exception') try: complex('foo', 1) except TypeError as e: assert str(e) == "complex() can't take second arg if first is a string" else: raise AssertionError('this was supposed to raise an exception') try: complex(1, 'bar') except TypeError as e: assert str(e) == "complex() second arg can't be a string" else: raise AssertionError('this was supposed to raise an exception') # __nonzero__ assert complex(0, 0).__nonzero__() == False assert complex(.0, .0).__nonzero__() == False assert complex(0.0, 0.1).__nonzero__() == True assert complex(1, 0).__nonzero__() == True assert complex(3.14, -0.001e+5).__nonzero__() == True assert complex(float('nan'), float('nan')).__nonzero__() == True assert complex(-float('inf'), float('inf')).__nonzero__() == True # __pos__ assert complex(0, 0).__pos__() == 0j assert complex(42, -0.1).__pos__() == (42-0.1j) assert complex(-1.2, 375E+2).__pos__() == (-1.2+37500j) assert repr(complex(5, float('nan')).__pos__()) == '(5+nanj)' assert repr(complex(float('inf'), 0.618).__pos__()) == '(inf+0.618j)' ================================================ FILE: testing/comprehension_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import types assert isinstance((x for x in ()), types.GeneratorType) assert list(c for c in 'abc') == ['a', 'b', 'c'] assert [c for c in 'abc'] == ['a', 'b', 'c'] assert [i + j for i in range(2) for j in range(2)] == [0, 1, 1, 2] assert [c for c in 'foobar' if c in 'aeiou'] == ['o', 'o', 'a'] assert {i: str(i) for i in range(2)} == {0: '0', 1: '1'} ================================================ FILE: testing/dict_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. d = {'foo': 1, 'bar': 2, 'baz': 3} try: d['qux'] except KeyError: pass assert d['foo'] == 1 assert d['bar'] == 2 assert d['baz'] == 3 d['qux'] = 4 assert d['qux'] == 4 d['foo'] = 5 assert d['foo'] == 5 l = [] for k in d: l.append(k) assert l == ['baz', 'foo', 'bar', 'qux'] try: for k in d: d['quux'] = 6 except RuntimeError: pass else: raise AssertionError d = {'foo': 1, 'bar': 2, 'baz': 3} del d['bar'] assert d == {'foo': 1, 'baz': 3} try: del d['bar'] except KeyError: pass else: raise AssertionError # Test clear d = {1: 1, 2: 2, 3: 3} d.clear() assert d == {} try: d.clear() assert AssertionError except TypeError: pass ================================================ FILE: testing/file_test.py ================================================ f = open('/tmp/file_test__someunlikelyexistingfile', 'w') assert f.softspace == 0 f.softspace = 1 assert f.softspace == 1 try: f.softspace = '4321' # should not be converted automatically except TypeError as e: if not str(e).endswith('is required'): raise e # Wrong exception arrived to us! else: raise RuntimeError('a TypeError should had raised.') assert f.softspace == 1 ================================================ FILE: testing/float_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. assert 5.5 == 5.5e0 assert 5. == 5.0 assert 0.5 == 5e-1 assert 1e6 == 1000000.0 assert 1E6 == 1e6 assert -1E6 == -1e6 assert 1E+6 == 1e6 assert 1E-6 == 0.000001 ================================================ FILE: testing/for_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. l = [] for i in (1, 2, 3): l.append(i) assert l == [1, 2, 3] l = [] for i in (): l.append(1) else: # pylint: disable=useless-else-on-loop l.append(2) assert l == [2] l = [] for i in (1,): l.append(i) else: # pylint: disable=useless-else-on-loop l.append(2) assert l == [1, 2] l = [] for i in (1,): l.append(i) break else: l.append(2) assert l == [1] l = [] for i in (1, 2): l.append(i) continue l.append(3) # pylint: disable=unreachable assert l == [1, 2] l = [] for i, j in [('a', 1), ('b', 2)]: l.append(i) l.append(j) assert l == ['a', 1, 'b', 2] # break and continue statements in an else clause applies to the outer loop. # See: https://github.com/google/grumpy/issues/123 l = [] for i in range(2): l.append(i) for j in range(10, 12): l.append(j) else: l.append(12) continue l.append(-1) assert l == [0, 10, 11, 12, 1, 10, 11, 12] l = [] for i in range(10): l.append(i) for j in range(10, 12): l.append(j) else: break l.append(-1) assert l == [0, 10, 11] ================================================ FILE: testing/function_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=no-value-for-parameter,function-redefined def foo(a): return {'a': a} assert foo(123) == {'a': 123} assert foo(a='apple') == {'a': 'apple'} assert foo(*('bar',)) == {'a': 'bar'} assert foo(**{'a': 42}) == {'a': 42} try: foo(b='bear') # pylint: disable=unexpected-keyword-arg raise AssertionError except TypeError as e: assert str(e) == "foo() got an unexpected keyword argument 'b'" try: foo() raise AssertionError except TypeError: pass try: foo(1, 2, 3) # pylint: disable=too-many-function-args raise AssertionError except TypeError: pass def foo(a, b): return {'a': a, 'b': b} assert foo(1, 2) == {'a': 1, 'b': 2} assert foo(1, b='bear') == {'a': 1, 'b': 'bear'} assert foo(b='bear', a='apple') == {'a': 'apple', 'b': 'bear'} try: foo(1, a='alpha') # pylint: disable=redundant-keyword-arg raise AssertionError except TypeError as e: assert str(e) == "foo() got multiple values for keyword argument 'a'" try: foo(**{123: 'bar'}) pass except TypeError: pass def foo(a, b=None): return {'a': a, 'b': b} assert foo(123) == {'a': 123, 'b': None} assert foo(123, 'bar') == {'a': 123, 'b': 'bar'} assert foo(a=123, b='bar') == {'a': 123, 'b': 'bar'} assert foo(*('apple',), **{'b': 'bear'}) == {'a': 'apple', 'b': 'bear'} def foo(a, *args): return {'a': a, 'args': args} assert foo(1) == {'a': 1, 'args': ()} assert foo(1, 2, 3) == {'a': 1, 'args': (2, 3)} def foo(a, **kwargs): return {'a': a, 'kwargs': kwargs} assert foo('bar') == {'a': 'bar', 'kwargs': {}} assert (foo(**{'a': 'apple', 'b': 'bear'}) == {'a': 'apple', 'kwargs': {'b': 'bear'}}) assert (foo('bar', b='baz', c='qux') == {'a': 'bar', 'kwargs': {'b': 'baz', 'c': 'qux'}}) ================================================ FILE: testing/generator_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import types def gen1(): yield 1 yield 2 yield 3 g = gen1() assert isinstance(g, types.GeneratorType) assert list(g) == [1, 2, 3] assert list(g) == [] # pylint: disable=g-explicit-bool-comparison def gen2(): for c in 'foobar': yield c yield '!' g = gen2() assert list(g) == ['f', 'o', 'o', 'b', 'a', 'r', '!'] assert list(g) == [] # pylint: disable=g-explicit-bool-comparison def gen3(): raise RuntimeError yield 1 # pylint: disable=unreachable g = gen3() try: g.next() except RuntimeError: pass assert list(g) == [] # pylint: disable=g-explicit-bool-comparison def gen4(): yield g.next() g = gen4() try: g.next() except ValueError as e: assert 'generator already executing' in str(e), str(e) else: raise AssertionError def gen5(): yield g = gen5() try: g.send('foo') except TypeError as e: assert "can't send non-None value to a just-started generator" in str(e) else: raise AssertionError def gen6(): yield 1 return yield 2 g = gen6() assert list(g) == [1] assert list(g) == [] ================================================ FILE: testing/getopt_test.py ================================================ import getopt args = '-a -b -cfoo -d bar a1 a2'.split() optlist, args = getopt.getopt(args, 'abc:d:') assert optlist == [('-a', ''), ('-b', ''), ('-c', 'foo'), ('-d', 'bar')] # TODO: str.index has to be implemented # s = '--condition=foo --testing --output-file abc.def -x a1 a2' # args = s.split() # optlist, args = getopt.getopt( # args, 'x', ['condition=', 'output-file=', 'testing']) # assert optlist == [('--condition', 'foo'), ('--testing', ''), # ('--output-file', 'abc.def'), ('-x', '')] ================================================ FILE: testing/global_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=g-wrong-blank-lines,global-variable-not-assigned,invalid-name,redefined-outer-name,unused-variable x = 123 def f1(): global x x = 'abc' f1() assert x == 'abc' x = 'foo' def f2(): global x class x(object): pass f2() assert isinstance(x, type) assert x.__name__ == 'x' x = 3.14 class C1(object): global x x = 'foo' assert x == 'foo' x = 42 def f3(): global x del x f3() try: print x raise AssertionError except NameError: pass x = 'foo' def f4(): x = 'bar' def g(): global x def h(): return x return h() return g() assert f4() == 'foo' x = 3.14 def f5(): x = 'foo' class C(object): global x y = x return C.y assert f5() == 3.14 ================================================ FILE: testing/if_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=using-constant-test foo = [] if True: foo.append(1) else: foo.append(2) assert foo == [1] foo = [] if False: foo.append(1) else: foo.append(2) assert foo == [2] foo = [] if False: foo.append(1) elif False: foo.append(2) elif True: foo.append(3) assert foo == [3] foo = [] if False: foo.append(1) elif True: foo.append(2) elif True: foo.append(3) else: foo.append(4) assert foo == [2] foo = [] if False: foo.append(1) elif False: foo.append(2) elif False: foo.append(3) else: foo.append(4) assert foo == [4] ================================================ FILE: testing/import_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys print sys.maxint ================================================ FILE: testing/list_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. a = [0, 1, 2, 3] b = list(a) assert a == b assert a is not b assert list(()) == [] assert list((0, 1, 2, 3)) == [0, 1, 2, 3] assert list('') == [] assert list('spam') == ['s', 'p', 'a', 'm'] assert [] is not True assert [42] assert [] is not [] assert len([]) == 0 assert len([0]) == 1 assert len([0, 1, 2]) == 3 a = [3, 2, 4, 1] b = [] c = ["a", "e", "c", "b"] a.sort() assert a == [1, 2, 3, 4] b.sort() assert b == [] c.sort() assert c == ["a", "b", "c", "e"] # Test pop a = [-1, 0, 1] assert a.pop() == 1 assert a == [-1, 0] assert a == [-1, 0] assert a.pop(0) == -1 assert a == [0] try: a.pop(5) assert AssertionError except IndexError: pass assert a.pop(0) == 0 assert a == [] try: a.pop() assert AssertionError except IndexError: pass try: a.pop(42, 42) assert AssertionError except TypeError: pass a = [-1, 0, 1] assert a.pop(1) == 0 assert a == [-1, 1] # Test extend a = aa = [3, 2, 4, 1] b = bb = [] c = cc = ["a", "e", "c", "b"] a.extend(b) assert a == [3, 2, 4, 1] assert a == aa assert a is aa b.extend(c) assert b == ["a", "e", "c", "b"] assert b is bb a.extend(tuple()) assert a == [3, 2, 4, 1] a.extend((6, 7)) assert a == [3, 2, 4, 1, 6, 7] a.extend(range(3)) assert a == [3, 2, 4, 1, 6, 7, 0, 1, 2] try: a.extend() assert AssertionError except TypeError: pass try: a.extend([], []) assert AssertionError except TypeError: pass # Test count assert [].count(0) == 0 assert [1, 2, 3].count(2) == 1 assert ["a", "b", "a", "a"].count("a") == 3 assert ([2] * 20).count(2) == 20 try: [].count() assert AssertionError except TypeError: pass ================================================ FILE: testing/native_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=g-multiple-import from '__go__/math' import MaxInt32, Pow10, Signbit from '__go__/strings' import Count, IndexAny, Repeat from '__go__/encoding/csv' import NewReader as NewCSVReader from '__go__/image' import Pt from '__go__/strings' import NewReader as NewStringReader assert Count('foo,bar,baz', ',') == 2 assert IndexAny('foobar', 'obr') == 1 assert Repeat('foo', 3) == 'foofoofoo' assert MaxInt32 == 2147483647 assert Pow10(2.0) == 100.0 assert Signbit(-42.0) == True # pylint: disable=g-explicit-bool-comparison # Can access field on unreferenced struct (Pt returns an image.Point struct) assert Pt(1, 0).X == 1 # Can access field on pointer to struct (NewCSVReader returns a pointer to a # csv.Reader struct) assert NewCSVReader(NewStringReader("foo")).LazyQuotes == False ================================================ FILE: testing/op_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Arithmetic and boolean operator tests.""" import math import weetest def TestBoolOps(): assert ('foo' or 'bar') == 'foo' assert ('' or 123) == 123 assert (0 and 3.14) == 0 assert (True and False) is False assert (0 or 'a' and 'b') == 'b' assert (1 and 'a' or 'b') == 'a' def TestBoolOpsLazyEval(): def Yes(): ran.append('Yes') return True def No(): ran.append('No') return False ran = [] assert Yes() or No() assert ran == ['Yes'] ran = [] assert not (Yes() and Yes() and No()) assert ran == ['Yes', 'Yes', 'No'] ran = [] assert not (Yes() and No() and Yes()) assert ran == ['Yes', 'No'] ran = [] assert No() or No() or Yes() assert ran == ['No', 'No', 'Yes'] ran = [] assert Yes() or Yes() or Yes() assert ran == ['Yes'] def TestNeg(): x = 12 assert -x == -12 x = 1.1 assert -x == -1.1 x = 0.0 assert -x == -0.0 x = float('inf') assert math.isinf(-x) x = -float('inf') assert math.isinf(-x) x = float('nan') assert math.isnan(-x) x = long(100) assert -x == -100 def TestPos(): x = 12 assert +x == 12 x = 1.1 assert +x == 1.1 x = 0.0 assert +x == 0.0 x = float('inf') assert math.isinf(+x) x = +float('inf') assert math.isinf(+x) x = float('nan') assert math.isnan(+x) x = long(100) assert +x == 100 if __name__ == '__main__': weetest.RunTests() ================================================ FILE: testing/pow_test.py ================================================ # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. assert 2.0 ** -2 == 0.25, "2.0 ** -2" assert 2.0 ** -1 == 0.5, "2.0 ** -1" assert 2.0 ** 0 == 1, "2.0 ** 0" assert 2.0 ** 1 == 2, "2.0 ** 1" assert 2.0 ** 2 == 4, "2.0 ** 2" assert (-2.0) ** -2 == 0.25, "(-2.0) ** -2" assert (-2.0) ** -1 == -0.5, "(-2.0) ** -1" assert (-2.0) ** 0 == 1, "(-2.0) ** 0" assert (-2.0) ** 1 == -2, "(-2.0) ** 1" assert (-2.0) ** 2 == 4, "(-2.0) ** 2" assert 2 ** -2 == 0.25, "2 ** -2" assert 2 ** -1 == 0.5, "2 ** -1" assert 2 ** 0 == 1, "2 ** 0" assert 2 ** 1 == 2, "2 ** 1" assert 2 ** 2 == 4, "2 ** 2" assert 2L ** -2 == 0.25, "2L ** -2" assert 2L ** -1 == 0.5, "2L ** -1" assert 2L ** 0 == 1, "2L ** 0" assert 2L ** 1 == 2, "2L ** 1" assert 2L ** 2 == 4, "2L ** 2" # Test the rpow operator on long assert 2 ** -2L == 0.25, "2 ** -2L" assert 2 ** -1L == 0.5, "2 ** -1L" assert 2 ** 0L == 1, "2 ** 0L" assert 2 ** 1L == 2, "2 ** 1L" assert 2 ** 2L == 4, "2 ** 2L" for zero in (0, 0L, 0.0): try: result = zero ** -2 assert "0 ** -2" except ZeroDivisionError: pass try: result = zero ** -1 assert "0 ** -1" except ZeroDivisionError: pass assert zero ** 0 == 1, '0 ** 0' assert zero ** 1 == 0, '0 ** 1' assert zero ** 2 == 0, '0 ** 2' assert 2 ** zero == 1 assert (-2.0) ** zero == 1 assert 3L ** zero == 1 assert (-2) ** -2 == 0.25, '(-2) ** -2' assert (-2) ** -1 == -0.5, '(-2) ** -1' assert (-2) ** 0 == 1, '(-2) ** 0' assert (-2) ** 1 == -2, '(-2) ** 1' assert (-2) ** 2 == 4, '(-2) ** 2' assert 2 ** 128 == 340282366920938463463374607431768211456, "2 ** 128" # chose something which can be represented exact as an IEEE floating point number large_number = (2 ** 128 + 2 ** 127) assert large_number ** -1 == (1.0 / large_number), "large_number ** -1 == (1.0 / large_number)" assert large_number ** 0 == 1, "large_number ** 0 == 1" assert large_number ** 1 == large_number, "large_number ** 1 == large_number" ================================================ FILE: testing/scope_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=redefined-outer-name x = 'foo' y = 'wut' assert x == 'foo' assert y == 'wut' def f(): x = 'bar' z = 'baz' assert x == 'bar' assert y == 'wut' assert z == 'baz' def g(arg): x = 'qux' assert x == 'qux' assert y == 'wut' assert z == 'baz' assert arg == 'quux' arg = None g('quux') f() # Delete a local var. def g(): foo = 'bar' del foo try: foo except UnboundLocalError: pass else: raise AssertionError g() # Delete a global. foo = 'bar' del foo try: foo except NameError: pass else: raise AssertionError # Delete a class var. class Foo(object): foo = 'bar' del foo try: foo except NameError: pass else: raise AssertionError ================================================ FILE: testing/str_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=redefined-outer-name,pointless-statement import sys # Test Add assert "foo" + "bar" == "foobar" assert "foo" + u"bar" == u"foobar" assert "baz" + "" == "baz" # Test capitalize assert "".capitalize() == "" assert "foo".capitalize() == "Foo" assert "Foo".capitalize() == "Foo" assert "FOO".capitalize() == "Foo" # Test count assert "".count("a") == 0 assert "abcd".count("e") == 0 assert "abccdef".count("c") == 2 assert "abba".count("bb") == 1 assert "abbba".count("bb") == 1 assert "abbbba".count("bb") == 2 assert "five".count("") == 5 assert ("a" * 20).count("a") == 20 try: "".count() assert AssertionError except TypeError: pass # Test find assert "".find("") == 0 assert "".find("", 1) == -1 assert "".find("", -1) == 0 assert "".find("", None, -1) == 0 assert "foobar".find("bar") == 3 assert "foobar".find("bar", 0, -2) == -1 assert "foobar".find("foo", 0, 3) == 0 assert "foobar".find("bar", 3, 5) == -1 assert "foobar".find("bar", 5, 3) == -1 assert 'foobar'.find("bar", None) == 3 assert 'foobar'.find("bar", 0, None) == 3 assert "bar".find("foobar") == -1 assert "bar".find("a", 0, -1) == 1 assert 'abcdefghiabc'.find('abc') == 0 assert 'abcdefghiabc'.find('abc', 1) == 9 assert 'abcdefghiabc'.find('def', 4) == -1 assert 'abc'.find('', 0) == 0 assert 'abc'.find('', 3) == 3 assert 'abc'.find('c', long(1)) == 2 assert 'abc'.find('c', 0, long(3)) == 2 assert 'abc'.find('', 4) == -1 assert 'rrarrrrrrrrra'.find('a') == 2 assert 'rrarrrrrrrrra'.find('a', 4) == 12 assert 'rrarrrrrrrrra'.find('a', 4, 6) == -1 assert 'rrarrrrrrrrra'.find('a', 4, None) == 12 assert 'rrarrrrrrrrra'.find('a', None, 6) == 2 assert ''.find('') == 0 assert ''.find('', 1, 1) == -1 assert ''.find('', sys.maxint, 0) == -1 assert ''.find('xx') == -1 assert ''.find('xx', 1, 1) == -1 assert ''.find('xx', sys.maxint, 0) == -1 assert 'ab'.find('xxx', sys.maxsize + 1, 0) == -1 # TODO: Support unicode substring. # assert "foobar".find(u"bar") == 3 class Foo(object): def __index__(self): return 3 assert 'abcd'.find('a', Foo()) == -1 try: "foo".find(123) raise AssertionError except TypeError: pass try: 'foo'.find() # pylint: disable=no-value-for-parameter raise AssertionError except TypeError: pass try: 'foo'.find(42) raise AssertionError except TypeError: pass try: 'foobar'.find("bar", "baz") raise AssertionError except TypeError: pass try: 'foobar'.find("bar", 0, "baz") raise AssertionError except TypeError: pass # Test GetItem class IntIndexType(object): def __index__(self): return 2 class LongIndexType(object): def __index__(self): return 2L class IntIntType(object): def __int__(self): return 2 class LongIntType(object): def __int__(self): return 2L assert "bar"[1] == "a" assert "bar"[long(1)] == "a" assert "baz"[-1] == "z" assert "baz"[IntIndexType()] == "z" assert "baz"[LongIndexType()] == "z" assert "bar"[None:2] == "ba" assert "bar"[1:3] == "ar" assert "bar"[1:None] == "ar" assert "foobarbaz"[1:8:2] == "obra" assert "abc"[None:None:-1] == "cba" try: "baz"[-4] raise AssertionError except IndexError: pass try: ""[0] raise AssertionError except IndexError: pass try: "foo"[3] raise AssertionError except IndexError: pass try: "foo"[3.14] #pylint: disable=invalid-sequence-index raise AssertionError except TypeError: pass try: "bar"[1:2:0] raise AssertionError except ValueError: pass # Test Mod assert "%s" % 42 == "42" assert "%f" % 3.14 == "3.140000" assert "abc %d" % 123L == "abc 123" assert "%d" % 3.14 == "3" assert "%%" % tuple() == "%" assert "%r" % "abc" == "'abc'" assert "%x" % 0x1f == "1f" assert "%X" % 0xffff == "FFFF" vals = [ ['-16', '-16', ' -16', '-16', '-000000016'], ['-10', '-10', ' -10', '-10', '-000000010'], ['-10', '-10', ' -10', '-10', '-000000010'], ['-16', '-16', ' -16', '-16', ' -16'], ['-16.000000', '-16.000000', '-16.000000', '-16.000000', '-16.000000'], ['-16', '-16', ' -16', '-16', ' -16'], ['-20', '-20', ' -20', '-20', '-000000020'], ['-10', '-10', ' -10', '-10', '-000000010'], ['-a', '-a', ' -a', '-a', '-00000000a'], ['-A', '-A', ' -A', '-A', '-00000000A'], ['-10', '-10', ' -10', '-10', ' -10'], ['-10.000000', '-10.000000', '-10.000000', '-10.000000', '-10.000000'], ['-10', '-10', ' -10', '-10', ' -10'], ['-12', '-12', ' -12', '-12', '-000000012'], ['-1', '-1', ' -1', '-1', '-000000001'], ['-1', '-1', ' -1', '-1', '-000000001'], ['-1', '-1', ' -1', '-1', '-000000001'], ['-1', '-1', ' -1', '-1', ' -1'], ['-1.000000', '-1.000000', ' -1.000000', '-1.000000', '-01.000000'], ['-1', '-1', ' -1', '-1', ' -1'], ['-1', '-1', ' -1', '-1', '-000000001'], ['0', ' 0', ' 0', '00', '0000000000'], ['0', ' 0', ' 0', '00', '0000000000'], ['0', ' 0', ' 0', '00', '0000000000'], ['0', ' 0', ' 0', ' 0', ' 0'], ['0.000000', '0.000000', ' 0.000000', '0.000000', '000.000000'], ['0', ' 0', ' 0', ' 0', ' 0'], ['0', ' 0', ' 0', '00', '0000000000'], ['1', ' 1', ' 1', '01', '0000000001'], ['1', ' 1', ' 1', '01', '0000000001'], ['1', ' 1', ' 1', '01', '0000000001'], ['1', ' 1', ' 1', ' 1', ' 1'], ['1.000000', '1.000000', ' 1.000000', '1.000000', '001.000000'], ['1', ' 1', ' 1', ' 1', ' 1'], ['1', ' 1', ' 1', '01', '0000000001'], ['3', ' 3', ' 3', '03', '0000000003'], ['3', ' 3', ' 3', '03', '0000000003'], ['3', ' 3', ' 3', '03', '0000000003'], ['3.14', '3.14', ' 3.14', '3.14', ' 3.14'], ['3.140000', '3.140000', ' 3.140000', '3.140000', '003.140000'], ['3.14', '3.14', ' 3.14', '3.14', ' 3.14'], ['3', ' 3', ' 3', '03', '0000000003'], ['10', '10', ' 10', '10', '0000000010'], ['a', ' a', ' a', '0a', '000000000a'], ['A', ' A', ' A', '0A', '000000000A'], ['10', '10', ' 10', '10', ' 10'], ['10.000000', '10.000000', ' 10.000000', '10.000000', '010.000000'], ['10', '10', ' 10', '10', ' 10'], ['12', '12', ' 12', '12', '0000000012'], ['16', '16', ' 16', '16', '0000000016'], ['10', '10', ' 10', '10', '0000000010'], ['10', '10', ' 10', '10', '0000000010'], ['16', '16', ' 16', '16', ' 16'], ['16.000000', '16.000000', ' 16.000000', '16.000000', '016.000000'], ['16', '16', ' 16', '16', ' 16'], ['20', '20', ' 20', '20', '0000000020'], ] i = 0 for a in [-16, -10, -1, 0, 1, 3.14, 10, 16]: for b in "dxXrfso": assert [("%" + b) % (a, ), ("%2" + b) % (a, ), ("%10" + b) % (a, ), ("%02" + b) % (a, ), ("%010" + b) % (a, )] == vals[i] i += 1 # Test replace assert 'one!two!three!'.replace('!', '@', 1) == 'one@two!three!' assert 'one!two!three!'.replace('!', '') == 'onetwothree' assert 'one!two!three!'.replace('!', '@', 2) == 'one@two@three!' assert 'one!two!three!'.replace('!', '@', 3) == 'one@two@three@' assert 'one!two!three!'.replace('!', '@', 4) == 'one@two@three@' assert 'one!two!three!'.replace('!', '@', 0) == 'one!two!three!' assert 'one!two!three!'.replace('!', '@') == 'one@two@three@' assert 'one!two!three!'.replace('x', '@') == 'one!two!three!' assert 'one!two!three!'.replace('x', '@', 2) == 'one!two!three!' assert 'abc'.replace('', '-') == '-a-b-c-' assert 'abc'.replace('', '-', 3) == '-a-b-c' assert 'abc'.replace('', '-', 0) == 'abc' assert ''.replace('', '') == '' assert ''.replace('', 'a') == 'a' assert 'abc'.replace('a', '--', 0) == 'abc' assert 'abc'.replace('xy', '--') == 'abc' assert '123'.replace('123', '') == '' assert '123123'.replace('123', '') == '' assert '123x123'.replace('123', '') == 'x' assert "\xd0\xb2\xd0\xbe\xd0\xbb".replace('', '\0') == "\x00\xd0\x00\xb2\x00\xd0\x00\xbe\x00\xd0\x00\xbb\x00" assert "\xd0\xb2\xd0\xbe\xd0\xbb".replace('', '\1\2') == '\x01\x02\xd0\x01\x02\xb2\x01\x02\xd0\x01\x02\xbe\x01\x02\xd0\x01\x02\xbb\x01\x02' class S(str): pass s = S('abc') assert type(s.replace(s, s)) is str assert type(s.replace('x', 'y')) is str assert type(s.replace('x', 'y', 0)) is str # CPython only, pypy supposed to be same as Go assert ''.replace('', 'x') == 'x' assert ''.replace('', 'x', -1) == 'x' assert ''.replace('', 'x', 0) == '' assert ''.replace('', 'x', 1) == '' assert ''.replace('', 'x', 1000) == '' try: ''.replace(None, '') raise AssertionError except TypeError: pass try: ''.replace('', None) raise AssertionError except TypeError: pass try: ''.replace('', '', None) raise AssertionError except TypeError: pass class A(object): def __int__(self): return 3 class AL(object): def __int__(self): return 3L class B(object): def __index__(self): return 3 class BL(object): def __index__(self): return 3L assert 'aaaaa'.replace('a', 'b', A()) == 'bbbaa' assert 'aaaaa'.replace('a', 'b', AL()) == 'bbbaa' try: 'aaaaa'.replace('a', 'b', B()) raise AssertionError except TypeError: pass try: 'aaaaa'.replace('a', 'b', BL()) raise AssertionError except TypeError: pass # Test split assert "".split() == [] assert " ".split() == [] assert "".split('x') == [''] assert "a".split() == ['a'] assert " ".split(" ", 1) == ['', ''] assert "aa".split("a", 2) == ['', '', ''] assert " a ".split() == ['a'] assert 'a b c d'.split(None, 1) == ['a', 'b c d'] assert 'a b c d '.split() == ['a', 'b', 'c', 'd'] assert ' a b c d '.split(None, 1) == ['a', 'b c d '] assert ' a b c d'.split(None, 0) == ['a b c d'] # Test zfill assert '123'.zfill(2) == '123' assert '123'.zfill(3) == '123' assert '123'.zfill(4) == '0123' assert '+123'.zfill(3) == '+123' assert '+123'.zfill(4) == '+123' assert '+123'.zfill(5) == '+0123' assert '-123'.zfill(3) == '-123' assert '-123'.zfill(4) == '-123' assert '-123'.zfill(5) == '-0123' assert ''.zfill(3) == '000' assert '34'.zfill(1) == '34' assert '34'.zfill(4) == '0034' try: '123'.zfill() raise AssertionError except TypeError: pass class A(object): def __int__(self): return 3 assert '3'.zfill(A()) == '003' assert '3'.zfill(IntIntType()) == '03' assert '3'.zfill(LongIntType()) == '03' assert '%o' % 8 == '10' assert '%o' % -8 == '-10' assert '%o %o' % (8, -8) == '10 -10' ================================================ FILE: testing/struct_test.py ================================================ # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import _struct as struct # struct test A = 0x67452301 B = 0xefcdab89 C = 0x98badcfe D = 0x10325476 expected = '\x01#Eg\x89\xab\xcd\xef\xfe\xdc\xba\x98vT2\x10' assert struct.pack("= b." return a >= b def gt(a, b): "Same as a > b." return a > b # Logical Operations **********************************************************# def not_(a): "Same as not a." return not a def truth(a): "Return True if a is true, False otherwise." return True if a else False def is_(a, b): "Same as a is b." return a is b def is_not(a, b): "Same as a is not b." return a is not b # Mathematical/Bitwise Operations *********************************************# def abs(a): "Same as abs(a)." return _abs(a) def add(a, b): "Same as a + b." return a + b def and_(a, b): "Same as a & b." return a & b def floordiv(a, b): "Same as a // b." return a // b def index(a): "Same as a.__index__()." return a.__index__() def inv(a): "Same as ~a." return ~a invert = inv def lshift(a, b): "Same as a << b." return a << b def mod(a, b): "Same as a % b." return a % b def mul(a, b): "Same as a * b." return a * b def neg(a): "Same as -a." return -a def or_(a, b): "Same as a | b." return a | b def pos(a): "Same as +a." return +a def pow(a, b): "Same as a ** b." return a**b def rshift(a, b): "Same as a >> b." return a >> b def sub(a, b): "Same as a - b." return a - b def truediv(a, b): "Same as a / b." if type(a) == int or type(a) == long: a = float(a) return a / b def xor(a, b): "Same as a ^ b." return a ^ b # Sequence Operations *********************************************************# def concat(a, b): "Same as a + b, for a and b sequences." if not hasattr(a, '__getitem__'): msg = "'%s' object can't be concatenated" % type(a).__name__ raise TypeError(msg) return a + b def contains(a, b): "Same as b in a (note reversed operands)." return b in a def countOf(a, b): "Return the number of times b occurs in a." count = 0 for i in a: if i == b: count += 1 return count def delitem(a, b): "Same as del a[b]." del a[b] def getitem(a, b): "Same as a[b]." return a[b] def indexOf(a, b): "Return the first index of b in a." for i, j in enumerate(a): if j == b: return i else: raise ValueError('sequence.index(x): x not in sequence') def setitem(a, b, c): "Same as a[b] = c." a[b] = c def length_hint(obj, default=0): """ Return an estimate of the number of items in obj. This is useful for presizing containers when building from an iterable. If the object supports len(), the result will be exact. Otherwise, it may over- or under-estimate by an arbitrary amount. The result will be an integer >= 0. """ if not isinstance(default, int): msg = ("'%s' object cannot be interpreted as an integer" % type(default).__name__) raise TypeError(msg) try: return len(obj) except TypeError: pass try: hint = type(obj).__length_hint__ except AttributeError: return default try: val = hint(obj) except TypeError: return default if val is NotImplemented: return default if not isinstance(val, int): msg = ('__length_hint__ must be integer, not %s' % type(val).__name__) raise TypeError(msg) if val < 0: msg = '__length_hint__() should return >= 0' raise ValueError(msg) return val # Generalized Lookup Objects **************************************************# # TODO: class attrgetter: class attrgetter(object): """ Return a callable object that fetches the given attribute(s) from its operand. After f = attrgetter('name'), the call f(r) returns r.name. After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date). After h = attrgetter('name.first', 'name.last'), the call h(r) returns (r.name.first, r.name.last). """ def __init__(self, attr, *attrs): if not attrs: if not isinstance(attr, str): raise TypeError('attribute name must be a string') names = attr.split('.') def func(obj): for name in names: obj = getattr(obj, name) return obj self._call = func else: getters = tuple(map(attrgetter, (attr,) + attrs)) def func(obj): return tuple(getter(obj) for getter in getters) self._call = func def __call__(self, obj): return self._call(obj) # TODO: class itemgetter: class itemgetter(object): """ Return a callable object that fetches the given item(s) from its operand. After f = itemgetter(2), the call f(r) returns r[2]. After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3]) """ def __init__(self, item, *items): if not items: def func(obj): return obj[item] self._call = func else: items = (item,) + items def func(obj): return tuple(obj[i] for i in items) self._call = func def __call__(self, obj): return self._call(obj) # TODO: class methodcaller: class methodcaller(object): """ Return a callable object that calls the given method on its operand. After f = methodcaller('name'), the call f(r) returns r.name(). After g = methodcaller('name', 'date', foo=1), the call g(r) returns r.name('date', foo=1). """ def __init__(*args, **kwargs): if len(args) < 2: msg = "methodcaller needs at least one argument, the method name" raise TypeError(msg) self = args[0] self._name = args[1] self._args = args[2:] self._kwargs = kwargs def __call__(self, obj): return getattr(obj, self._name)(*self._args, **self._kwargs) # In-place Operations *********************************************************# def iadd(a, b): "Same as a += b." a += b return a def iand(a, b): "Same as a &= b." a &= b return a def iconcat(a, b): "Same as a += b, for a and b sequences." if not hasattr(a, '__getitem__'): msg = "'%s' object can't be concatenated" % type(a).__name__ raise TypeError(msg) a += b return a def ifloordiv(a, b): "Same as a //= b." a //= b return a def ilshift(a, b): "Same as a <<= b." a <<= b return a def imod(a, b): "Same as a %= b." a %= b return a def imul(a, b): "Same as a *= b." a *= b return a def ior(a, b): "Same as a |= b." a |= b return a def ipow(a, b): "Same as a **= b." a **= b return a def irshift(a, b): "Same as a >>= b." a >>= b return a def isub(a, b): "Same as a -= b." a -= b return a def itruediv(a, b): "Same as a /= b." if type(a) == int or type(a) == long: a = float(a) a /= b return a def ixor(a, b): "Same as a ^= b." a ^= b return a # TODO: https://github.com/google/grumpy/pull/263 #try: # from _operator import * #except ImportError: # pass #else: # from _operator import __doc__ # All of these "__func__ = func" assignments have to happen after importing # from _operator to make sure they're set to the right function __lt__ = lt __le__ = le __eq__ = eq __ne__ = ne __ge__ = ge __gt__ = gt __not__ = not_ __abs__ = abs __add__ = add __and__ = and_ __floordiv__ = floordiv __index__ = index __inv__ = inv __invert__ = invert __lshift__ = lshift __mod__ = mod __mul__ = mul __neg__ = neg __or__ = or_ __pos__ = pos __pow__ = pow __rshift__ = rshift __sub__ = sub __truediv__ = truediv __xor__ = xor __concat__ = concat __contains__ = contains __delitem__ = delitem __getitem__ = getitem __setitem__ = setitem __iadd__ = iadd __iand__ = iand __iconcat__ = iconcat __ifloordiv__ = ifloordiv __ilshift__ = ilshift __imod__ = imod __imul__ = imul __ior__ = ior __ipow__ = ipow __irshift__ = irshift __isub__ = isub __itruediv__ = itruediv __ixor__ = ixor ================================================ FILE: third_party/ouroboros/test/test_operator.py ================================================ import unittest import operator from test import test_support class Seq1(object): def __init__(self, lst): self.lst = lst def __len__(self): return len(self.lst) def __getitem__(self, i): return self.lst[i] def __add__(self, other): return self.lst + other.lst def __mul__(self, other): return self.lst * other def __rmul__(self, other): return other * self.lst class Seq2(object): def __init__(self, lst): self.lst = lst def __len__(self): return len(self.lst) def __getitem__(self, i): return self.lst[i] def __add__(self, other): return self.lst + other.lst def __mul__(self, other): return self.lst * other def __rmul__(self, other): return other * self.lst class OperatorTestCase(unittest.TestCase): def test_lt(self): #operator = self.module self.assertRaises(TypeError, operator.lt) self.assertFalse(operator.lt(1, 0)) self.assertFalse(operator.lt(1, 0.0)) self.assertFalse(operator.lt(1, 1)) self.assertFalse(operator.lt(1, 1.0)) self.assertTrue(operator.lt(1, 2)) self.assertTrue(operator.lt(1, 2.0)) def test_le(self): #operator = self.module self.assertRaises(TypeError, operator.le) self.assertFalse(operator.le(1, 0)) self.assertFalse(operator.le(1, 0.0)) self.assertTrue(operator.le(1, 1)) self.assertTrue(operator.le(1, 1.0)) self.assertTrue(operator.le(1, 2)) self.assertTrue(operator.le(1, 2.0)) def test_eq(self): #operator = self.module class C(object): def __eq__(self, other): raise SyntaxError self.assertRaises(TypeError, operator.eq) self.assertRaises(SyntaxError, operator.eq, C(), C()) self.assertFalse(operator.eq(1, 0)) self.assertFalse(operator.eq(1, 0.0)) self.assertTrue(operator.eq(1, 1)) self.assertTrue(operator.eq(1, 1.0)) self.assertFalse(operator.eq(1, 2)) self.assertFalse(operator.eq(1, 2.0)) def test_ne(self): #operator = self.module class C(object): def __ne__(self, other): raise SyntaxError self.assertRaises(TypeError, operator.ne) self.assertRaises(SyntaxError, operator.ne, C(), C()) self.assertTrue(operator.ne(1, 0)) self.assertTrue(operator.ne(1, 0.0)) self.assertFalse(operator.ne(1, 1)) self.assertFalse(operator.ne(1, 1.0)) self.assertTrue(operator.ne(1, 2)) self.assertTrue(operator.ne(1, 2.0)) def test_ge(self): #operator = self.module self.assertRaises(TypeError, operator.ge) self.assertTrue(operator.ge(1, 0)) self.assertTrue(operator.ge(1, 0.0)) self.assertTrue(operator.ge(1, 1)) self.assertTrue(operator.ge(1, 1.0)) self.assertFalse(operator.ge(1, 2)) self.assertFalse(operator.ge(1, 2.0)) def test_gt(self): #operator = self.module self.assertRaises(TypeError, operator.gt) self.assertTrue(operator.gt(1, 0)) self.assertTrue(operator.gt(1, 0.0)) self.assertFalse(operator.gt(1, 1)) self.assertFalse(operator.gt(1, 1.0)) self.assertFalse(operator.gt(1, 2)) self.assertFalse(operator.gt(1, 2.0)) def test_abs(self): #operator = self.module self.assertRaises(TypeError, operator.abs) self.assertRaises(TypeError, operator.abs, None) self.assertEqual(operator.abs(-1), 1) self.assertEqual(operator.abs(1), 1) def test_add(self): #operator = self.module self.assertRaises(TypeError, operator.add) self.assertRaises(TypeError, operator.add, None, None) self.assertTrue(operator.add(3, 4) == 7) def test_bitwise_and(self): #operator = self.module self.assertRaises(TypeError, operator.and_) self.assertRaises(TypeError, operator.and_, None, None) self.assertTrue(operator.and_(0xf, 0xa) == 0xa) def test_concat(self): #operator = self.module self.assertRaises(TypeError, operator.concat) self.assertRaises(TypeError, operator.concat, None, None) self.assertTrue(operator.concat('py', 'thon') == 'python') self.assertTrue(operator.concat([1, 2], [3, 4]) == [1, 2, 3, 4]) self.assertTrue(operator.concat(Seq1([5, 6]), Seq1([7])) == [5, 6, 7]) self.assertTrue(operator.concat(Seq2([5, 6]), Seq2([7])) == [5, 6, 7]) self.assertRaises(TypeError, operator.concat, 13, 29) def test_countOf(self): #operator = self.module self.assertRaises(TypeError, operator.countOf) self.assertRaises(TypeError, operator.countOf, None, None) self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 3) == 1) self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 5) == 0) @unittest.expectedFailure def test_delitem(self): #operator = self.module a = [4, 3, 2, 1] self.assertRaises(TypeError, operator.delitem, a) self.assertRaises(TypeError, operator.delitem, a, None) self.assertTrue(operator.delitem(a, 1) is None) self.assertTrue(a == [4, 2, 1]) def test_floordiv(self): #operator = self.module self.assertRaises(TypeError, operator.floordiv, 5) self.assertRaises(TypeError, operator.floordiv, None, None) self.assertTrue(operator.floordiv(5, 2) == 2) def test_truediv(self): #operator = self.module self.assertRaises(TypeError, operator.truediv, 5) self.assertRaises(TypeError, operator.truediv, None, None) self.assertTrue(operator.truediv(5, 2) == 2.5) def test_getitem(self): #operator = self.module a = range(10) self.assertRaises(TypeError, operator.getitem) self.assertRaises(TypeError, operator.getitem, a, None) self.assertTrue(operator.getitem(a, 2) == 2) def test_indexOf(self): #operator = self.module self.assertRaises(TypeError, operator.indexOf) self.assertRaises(TypeError, operator.indexOf, None, None) self.assertTrue(operator.indexOf([4, 3, 2, 1], 3) == 1) self.assertRaises(ValueError, operator.indexOf, [4, 3, 2, 1], 0) def test_invert(self): #operator = self.module self.assertRaises(TypeError, operator.invert) self.assertRaises(TypeError, operator.invert, None) self.assertEqual(operator.inv(4), -5) def test_lshift(self): #operator = self.module self.assertRaises(TypeError, operator.lshift) self.assertRaises(TypeError, operator.lshift, None, 42) self.assertTrue(operator.lshift(5, 1) == 10) self.assertTrue(operator.lshift(5, 0) == 5) self.assertRaises(ValueError, operator.lshift, 2, -1) def test_mod(self): #operator = self.module self.assertRaises(TypeError, operator.mod) self.assertRaises(TypeError, operator.mod, None, 42) self.assertTrue(operator.mod(5, 2) == 1) def test_mul(self): #operator = self.module self.assertRaises(TypeError, operator.mul) self.assertRaises(TypeError, operator.mul, None, None) self.assertTrue(operator.mul(5, 2) == 10) def test_neg(self): #operator = self.module self.assertRaises(TypeError, operator.neg) self.assertRaises(TypeError, operator.neg, None) self.assertEqual(operator.neg(5), -5) self.assertEqual(operator.neg(-5), 5) self.assertEqual(operator.neg(0), 0) self.assertEqual(operator.neg(-0), 0) def test_bitwise_or(self): #operator = self.module self.assertRaises(TypeError, operator.or_) self.assertRaises(TypeError, operator.or_, None, None) self.assertTrue(operator.or_(0xa, 0x5) == 0xf) def test_pos(self): #operator = self.module self.assertRaises(TypeError, operator.pos) self.assertRaises(TypeError, operator.pos, None) self.assertEqual(operator.pos(5), 5) self.assertEqual(operator.pos(-5), -5) self.assertEqual(operator.pos(0), 0) self.assertEqual(operator.pos(-0), 0) def test_pow(self): #operator = self.module self.assertRaises(TypeError, operator.pow) self.assertRaises(TypeError, operator.pow, None, None) self.assertEqual(operator.pow(3,5), 3**5) self.assertRaises(TypeError, operator.pow, 1) self.assertRaises(TypeError, operator.pow, 1, 2, 3) def test_rshift(self): #operator = self.module self.assertRaises(TypeError, operator.rshift) self.assertRaises(TypeError, operator.rshift, None, 42) self.assertTrue(operator.rshift(5, 1) == 2) self.assertTrue(operator.rshift(5, 0) == 5) self.assertRaises(ValueError, operator.rshift, 2, -1) def test_contains(self): #operator = self.module self.assertRaises(TypeError, operator.contains) self.assertRaises(TypeError, operator.contains, None, None) self.assertTrue(operator.contains(range(4), 2)) self.assertFalse(operator.contains(range(4), 5)) def test_setitem(self): #operator = self.module a = list(range(3)) self.assertRaises(TypeError, operator.setitem, a) self.assertRaises(TypeError, operator.setitem, a, None, None) self.assertTrue(operator.setitem(a, 0, 2) is None) self.assertTrue(a == [2, 1, 2]) self.assertRaises(IndexError, operator.setitem, a, 4, 2) def test_sub(self): #operator = self.module self.assertRaises(TypeError, operator.sub) self.assertRaises(TypeError, operator.sub, None, None) self.assertTrue(operator.sub(5, 2) == 3) @unittest.expectedFailure def test_truth(self): #operator = self.module class C(object): def __bool__(self): raise SyntaxError self.assertRaises(TypeError, operator.truth) self.assertRaises(SyntaxError, operator.truth, C()) self.assertTrue(operator.truth(5)) self.assertTrue(operator.truth([0])) self.assertFalse(operator.truth(0)) self.assertFalse(operator.truth([])) def test_bitwise_xor(self): #operator = self.module self.assertRaises(TypeError, operator.xor) self.assertRaises(TypeError, operator.xor, None, None) self.assertTrue(operator.xor(0xb, 0xc) == 0x7) def test_is(self): #operator = self.module a = b = 'xyzpdq' c = a[:3] + b[3:] self.assertRaises(TypeError, operator.is_) self.assertTrue(operator.is_(a, b)) #self.assertFalse(operator.is_(a,c)) @unittest.expectedFailure def test_is_not(self): #operator = self.module a = b = 'xyzpdq' c = a[:3] + b[3:] self.assertRaises(TypeError, operator.is_not) self.assertFalse(operator.is_not(a, b)) self.assertTrue(operator.is_not(a,c)) @unittest.expectedFailure def test_attrgetter(self): #operator = self.module class A(object): pass a = A() a.name = 'arthur' f = operator.attrgetter('name') self.assertEqual(f(a), 'arthur') f = operator.attrgetter('rank') self.assertRaises(AttributeError, f, a) self.assertRaises(TypeError, operator.attrgetter, 2) self.assertRaises(TypeError, operator.attrgetter) # multiple gets record = A() record.x = 'X' record.y = 'Y' record.z = 'Z' self.assertEqual(operator.attrgetter('x','z','y')(record), ('X', 'Z', 'Y')) self.assertRaises(TypeError, operator.attrgetter, ('x', (), 'y')) class C(object): def __getattr__(self, name): raise SyntaxError self.assertRaises(SyntaxError, operator.attrgetter('foo'), C()) # recursive gets a = A() a.name = 'arthur' a.child = A() a.child.name = 'thomas' f = operator.attrgetter('child.name') self.assertEqual(f(a), 'thomas') self.assertRaises(AttributeError, f, a.child) f = operator.attrgetter('name', 'child.name') self.assertEqual(f(a), ('arthur', 'thomas')) f = operator.attrgetter('name', 'child.name', 'child.child.name') self.assertRaises(AttributeError, f, a) f = operator.attrgetter('child.') self.assertRaises(AttributeError, f, a) f = operator.attrgetter('.child') self.assertRaises(AttributeError, f, a) a.child.child = A() a.child.child.name = 'johnson' f = operator.attrgetter('child.child.name') self.assertEqual(f(a), 'johnson') f = operator.attrgetter('name', 'child.name', 'child.child.name') self.assertEqual(f(a), ('arthur', 'thomas', 'johnson')) @unittest.expectedFailure def test_itemgetter(self): #operator = self.module a = 'ABCDE' f = operator.itemgetter(2) self.assertEqual(f(a), 'C') f = operator.itemgetter(10) self.assertRaises(IndexError, f, a) class C(object): def __getitem__(self, name): raise SyntaxError self.assertRaises(SyntaxError, operator.itemgetter(42), C()) f = operator.itemgetter('name') self.assertRaises(TypeError, f, a) self.assertRaises(TypeError, operator.itemgetter) d = dict(key='val') f = operator.itemgetter('key') self.assertEqual(f(d), 'val') f = operator.itemgetter('nonkey') self.assertRaises(KeyError, f, d) # example used in the docs inventory = [('apple', 3), ('banana', 2), ('pear', 5), ('orange', 1)] getcount = operator.itemgetter(1) self.assertEqual(list(map(getcount, inventory)), [3, 2, 5, 1]) self.assertEqual(sorted(inventory, key=getcount), [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)]) # multiple gets data = list(map(str, range(20))) self.assertEqual(operator.itemgetter(2,10,5)(data), ('2', '10', '5')) self.assertRaises(TypeError, operator.itemgetter(2, 'x', 5), data) def test_methodcaller(self): #operator = self.module self.assertRaises(TypeError, operator.methodcaller) class A(object): def foo(self, *args, **kwds): return args[0] + args[1] def bar(self, f=42): return f def baz(*args, **kwds): return kwds['name'], kwds['self'] a = A() f = operator.methodcaller('foo') self.assertRaises(IndexError, f, a) f = operator.methodcaller('foo', 1, 2) self.assertEqual(f(a), 3) f = operator.methodcaller('bar') self.assertEqual(f(a), 42) self.assertRaises(TypeError, f, a, a) f = operator.methodcaller('bar', f=5) self.assertEqual(f(a), 5) f = operator.methodcaller('baz', name='spam', self='eggs') self.assertEqual(f(a), ('spam', 'eggs')) @unittest.expectedFailure def test_inplace(self): #operator = self.module class C(object): def __iadd__ (self, other): return "iadd" def __iand__ (self, other): return "iand" def __ifloordiv__(self, other): return "ifloordiv" def __ilshift__ (self, other): return "ilshift" def __imod__ (self, other): return "imod" def __imul__ (self, other): return "imul" def __ior__ (self, other): return "ior" def __ipow__ (self, other): return "ipow" def __irshift__ (self, other): return "irshift" def __isub__ (self, other): return "isub" def __itruediv__ (self, other): return "itruediv" def __ixor__ (self, other): return "ixor" def __getitem__(self, other): return 5 # so that C is a sequence c = C() self.assertEqual(operator.iadd (c, 5), "iadd") self.assertEqual(operator.iand (c, 5), "iand") self.assertEqual(operator.ifloordiv(c, 5), "ifloordiv") self.assertEqual(operator.ilshift (c, 5), "ilshift") self.assertEqual(operator.imod (c, 5), "imod") self.assertEqual(operator.imul (c, 5), "imul") self.assertEqual(operator.ior (c, 5), "ior") self.assertEqual(operator.ipow (c, 5), "ipow") self.assertEqual(operator.irshift (c, 5), "irshift") self.assertEqual(operator.isub (c, 5), "isub") self.assertEqual(operator.itruediv (c, 5), "itruediv") self.assertEqual(operator.ixor (c, 5), "ixor") self.assertEqual(operator.iconcat (c, c), "iadd") @unittest.expectedFailure def test_length_hint(self): #operator = self.module class X(object): def __init__(self, value): self.value = value def __length_hint__(self): if type(self.value) is type: raise self.value else: return self.value self.assertEqual(operator.length_hint([], 2), 0) self.assertEqual(operator.length_hint(iter([1, 2, 3])), 3) self.assertEqual(operator.length_hint(X(2)), 2) self.assertEqual(operator.length_hint(X(NotImplemented), 4), 4) self.assertEqual(operator.length_hint(X(TypeError), 12), 12) with self.assertRaises(TypeError): operator.length_hint(X("abc")) with self.assertRaises(ValueError): operator.length_hint(X(-2)) with self.assertRaises(LookupError): operator.length_hint(X(LookupError)) def test_dunder_is_original(self): #operator = self.module names = [name for name in dir(operator) if not name.startswith('_')] for name in names: orig = getattr(operator, name) dunder = getattr(operator, '__' + name.strip('_') + '__', None) if dunder: self.assertIs(dunder, orig) def test_complex_operator(self): self.assertRaises(TypeError, operator.lt, 1j, 2j) self.assertRaises(TypeError, operator.le, 1j, 2j) self.assertRaises(TypeError, operator.ge, 1j, 2j) self.assertRaises(TypeError, operator.gt, 1j, 2j) def test_main(): test_support.run_unittest(OperatorTestCase) if __name__ == "__main__": test_main() ================================================ FILE: third_party/pypy/LICENSE ================================================ #encoding utf-8 License ======= Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation in the 'rpython', 'pypy', 'ctype_configure', 'dotviewer', 'demo', 'lib_pypy', 'py', and '_pytest' directories is licensed as follows: The MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PyPy Copyright holders 2003-2017 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at the beginning of each file) the files in the 'pypy' directory are each copyrighted by one or more of the following people and organizations: Armin Rigo Maciej Fijalkowski Carl Friedrich Bolz Amaury Forgeot d'Arc Antonio Cuni Samuele Pedroni Matti Picus Ronan Lamy Alex Gaynor Philip Jenvey Brian Kearns Richard Plangger Michael Hudson Manuel Jacob David Schneider Holger Krekel Christian Tismer Hakan Ardo Benjamin Peterson Anders Chrigstrom Wim Lavrijsen Eric van Riet Paap Richard Emslie Alexander Schremmer Remi Meier Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager Anders Lehmann Aurelien Campeas Niklaus Haldimann Camillo Bruni Laura Creighton Romain Guillebert Toon Verwaest Leonardo Santagada Seo Sanghyeon Ronny Pfannschmidt Justin Peel Raffael Tfirst David Edelsohn Anders Hammarquist Jakub Gustak Gregor Wegberg Guido Wesdorp Lawrence Oluyede Bartosz Skowron Daniel Roberts Adrien Di Mascio Niko Matsakis Alexander Hesse Ludovic Aubry Jacob Hallen Jason Creighton Mark Young Alex Martelli Spenser Bauman Michal Bendowski stian Jan de Mooij Tyler Wade Vincent Legoll Michael Foord Stephan Diehl Stefan Schwarzer Tomek Meka Valentino Volonghi Stefano Rivera Patrick Maupin Devin Jeanpierre Bob Ippolito Bruno Gola David Malcolm Jean-Paul Calderone Edd Barrett Squeaky Timo Paulssen Marius Gedminas Alexandre Fayolle Simon Burton Nicolas Truessel Martin Matusiak Wenzhu Man Konstantin Lopuhin John Witulski Laurence Tratt Greg Price Ivan Sichmann Freitas Dario Bertini Jeremy Thurgood Mark Pearse Simon Cross Tobias Pape Andreas Stührk Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov Paweł Piotr Przeradowski William Leslie marky1991 Ilya Osadchiy Tobias Oberstein Paul deGrandis Boris Feigin Taavi Burns Adrian Kuhn tav Georg Brandl Bert Freudenberg Stian Andreassen Wanja Saatkamp Mike Blume Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller Joannah Nanjekye Eugene Oden Tim Felgentreff Jeff Terrace Henry Mason Vasily Kuznetsov Preston Timmons David Ripton Dusty Phillips Lukas Renggli Guenter Jantzen Ned Batchelder Amit Regmi Anton Gulenko Sergey Matyunin Jasper Schulz Andrew Chambers Nicolas Chauvat Andrew Durdin Ben Young Michael Schneider Nicholas Riley Jason Chu Igor Trindade Oliveira Yichao Yu Michael Twomey Rocco Moretti Gintautas Miliauskas Lucian Branescu Mihaila anatoly techtonik Karl Bartel Gabriel Lavoie Jared Grubb Olivier Dormond Wouter van Heyst Sebastian Pawluś Brian Dorsey Victor Stinner Andrews Medina Aaron Iles Toby Watson Daniel Patrick Stuart Williams Antoine Pitrou Christian Hudon Justas Sadzevicius Neil Shepperd Michael Cheng Mikael Schönenberg Stanislaw Halik Berkin Ilbeyi Gasper Zejn Faye Zhao Elmo Mäntynen Anders Qvist Corbin Simpson Chirag Jadwani Jonathan David Riehl Beatrice During Alex Perry p_zieschang@yahoo.de Robert Zaremba Alan McIntyre Alexander Sedov Vaibhav Sood Reuben Cummings Attila Gobi Christopher Pope Tristan Arthur Christian Tismer Dan Stromberg Carl Meyer Florin Papa Valentina Mukhamedzhanova Stefano Parmesan touilleMan Marc Abramowitz Arjun Naik Aaron Gallagher Alexis Daboville Pieter Zieschang Karl Ramm Lukas Vacek Omer Katz Jacek Generowicz Sylvain Thenault Jakub Stasiak Stefan Beyer Andrew Dalke Alejandro J. Cura Vladimir Kryachko Gabriel Mark Williams Kunal Grover Nathan Taylor Travis Francis Athougies Yasir Suhail Sergey Kishchenko Martin Blais Lutz Paelike Ian Foote Philipp Rustemeuer Catalin Gabriel Manciu Jacob Oscarson Ryan Gonzalez Kristjan Valur Jonsson Lucio Torre Richard Lancaster Dan Buch Lene Wagner Tomo Cocoa Alecsandru Patrascu David Lievens Neil Blakey-Milner Henrik Vendelbo Lars Wassermann Ignas Mikalajunas Christoph Gerum Miguel de Val Borro Artur Lisiecki Toni Mattis Laurens Van Houtven Bobby Impollonia Roberto De Ioris Jeong YunWon Christopher Armstrong Aaron Tubbs Vasantha Ganesh K Jason Michalski Markus Holtermann Andrew Thompson Yusei Tahara Ruochen Huang Fabio Niephaus Akira Li Gustavo Niemeyer Rafał Gałczyński Logan Chien Lucas Stadler roberto@goyle Matt Bogosian Yury V. Zaytsev florinpapa Anders Sigfridsson Nikolay Zinov rafalgalczynski@gmail.com Joshua Gilbert Anna Katrina Dominguez Kim Jin Su Amber Brown Ben Darnell Juan Francisco Cantero Hurtado Godefroid Chappelle Julian Berman Michael Hudson-Doyle Floris Bruynooghe Stephan Busemann Dan Colish timo Volodymyr Vladymyrov Daniel Neuhäuser Flavio Percoco halgari Jim Baker Chris Lambacher coolbutuseless@gmail.com Mike Bayer Rodrigo Araújo Daniil Yarancev OlivierBlanvillain Jonas Pfannschmidt Zearin Andrey Churin Dan Crosta reubano@gmail.com Julien Phalip Roman Podoliaka Eli Stevens Boglarka Vezer PavloKapyshin Tomer Chachamu Christopher Groskopf Asmo Soinio Antony Lee Jim Hunziker shoma hosaka Buck Golemon JohnDoe yrttyr Michael Chermside Anna Ravencroft remarkablerocket Berker Peksag Christian Muirhead soareschen Matthew Miller Konrad Delong Dinu Gherman pizi James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon aliceinwire Zooko Wilcox-O Hearn James Lan jiaaro Markus Unterwaditzer Kristoffer Kleine Graham Markall Dan Loewenherz werat Niclas Olofsson Chris Pressey Tobias Diaz Nikolaos-Digenis Karagiannis Kurt Griffiths Ben Mather Donald Stufft Dan Sanders Jason Madden Yaroslav Fedevych Even Wiik Thomassen Stefan Marr Heinrich-Heine University, Germany Open End AB (formerly AB Strakt), Sweden merlinux GmbH, Germany tismerysoft GmbH, Germany Logilab Paris, France DFKI GmbH, Germany Impara, Germany Change Maker, Sweden University of California Berkeley, USA Google Inc. King's College London The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike License. License for 'lib-python/2.7' ============================ Except when otherwise stated (look for LICENSE files or copyright/license information at the beginning of each file) the files in the 'lib-python/2.7' directory are all copyrighted by the Python Software Foundation and licensed under the terms that you can find here: https://docs.python.org/2/license.html License for 'pypy/module/unicodedata/' ====================================== The following files are from the website of The Unicode Consortium at http://www.unicode.org/. For the terms of use of these files, see http://www.unicode.org/terms_of_use.html . Or they are derived from files from the above website, and the same terms of use apply. CompositionExclusions-*.txt EastAsianWidth-*.txt LineBreak-*.txt UnicodeData-*.txt UnihanNumeric-*.txt License for 'dotviewer/font/' ============================= Copyright (C) 2008 The Android Open Source Project Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Detailed license information is contained in the NOTICE file in the directory. Licenses and Acknowledgements for Incorporated Software ======================================================= This section is an incomplete, but growing list of licenses and acknowledgements for third-party software incorporated in the PyPy distribution. License for 'Tcl/Tk' -------------------- This copy of PyPy contains library code that may, when used, result in the Tcl/Tk library to be loaded. PyPy also includes code that may be regarded as being a copy of some parts of the Tcl/Tk header files. You may see a copy of the License for Tcl/Tk in the file `lib_pypy/_tkinter/license.terms` included here. License for 'bzip2' ------------------- This copy of PyPy may be linked (dynamically or statically) with the bzip2 library. You may see a copy of the License for bzip2/libbzip2 at http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html License for 'openssl' --------------------- This copy of PyPy may be linked (dynamically or statically) with the openssl library. You may see a copy of the License for OpenSSL at https://www.openssl.org/source/license.html License for 'gdbm' ------------------ The gdbm module includes code from gdbm.h, which is distributed under the terms of the GPL license version 2 or any later version. Thus the gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed under the terms of the GPL license as well. License for 'rpython/rlib/rvmprof/src' -------------------------------------- The code is based on gperftools. You may see a copy of the License for it at https://github.com/gperftools/gperftools/blob/master/COPYING ================================================ FILE: third_party/pypy/README.md ================================================ Canonical versions of the files in this folder come from the [lib-python/2.7](https://bitbucket.org/pypy/pypy/src/23fd2966aada422b331d7d752fc383178deffb27/lib-python/2.7/?at=default) directory of the [PyPy repo](https://bitbucket.org/pypy/pypy). ================================================ FILE: third_party/pypy/_collections.py ================================================ """High performance data structures """ # # Copied and completed from the sandbox of CPython # (nondist/sandbox/collections/pydeque.py rev 1.1, Raymond Hettinger) # # Note that PyPy also contains a built-in module '_collections' which will hide # this one if compiled in. # try: # from threading import _get_ident as _thread_ident # except ImportError: # def _thread_ident(): # return -1 import thread _thread_ident = thread.get_ident n = 30 LFTLNK = n RGTLNK = n+1 BLOCKSIZ = n+2 # The deque's size limit is d.maxlen. The limit can be zero or positive, or # None. After an item is added to a deque, we check to see if the size has # grown past the limit. If it has, we get the size back down to the limit by # popping an item off of the opposite end. The methods that can trigger this # are append(), appendleft(), extend(), and extendleft(). class deque(object): def __new__(cls, iterable=(), *args, **kw): self = super(deque, cls).__new__(cls) self.clear() return self def __init__(self, iterable=(), maxlen=None): self.clear() if maxlen is not None: if maxlen < 0: raise ValueError("maxlen must be non-negative") self._maxlen = maxlen add = self.append for elem in iterable: add(elem) # @property def maxlen(self): return self._maxlen # TODO: Make this a decorator once they're implemented. maxlen = property(maxlen) def clear(self): self.right = self.left = [None] * BLOCKSIZ self.rightndx = n//2 # points to last written element self.leftndx = n//2+1 self.length = 0 self.state = 0 def append(self, x): self.state += 1 self.rightndx += 1 if self.rightndx == n: newblock = [None] * BLOCKSIZ self.right[RGTLNK] = newblock newblock[LFTLNK] = self.right self.right = newblock self.rightndx = 0 self.length += 1 self.right[self.rightndx] = x if self.maxlen is not None and self.length > self.maxlen: self.popleft() def appendleft(self, x): self.state += 1 self.leftndx -= 1 if self.leftndx == -1: newblock = [None] * BLOCKSIZ self.left[LFTLNK] = newblock newblock[RGTLNK] = self.left self.left = newblock self.leftndx = n-1 self.length += 1 self.left[self.leftndx] = x if self.maxlen is not None and self.length > self.maxlen: self.pop() def extend(self, iterable): if iterable is self: iterable = list(iterable) for elem in iterable: self.append(elem) def extendleft(self, iterable): if iterable is self: iterable = list(iterable) for elem in iterable: self.appendleft(elem) def pop(self): if self.left is self.right and self.leftndx > self.rightndx: raise IndexError("pop from an empty deque") x = self.right[self.rightndx] self.right[self.rightndx] = None self.length -= 1 self.rightndx -= 1 self.state += 1 if self.rightndx == -1: prevblock = self.right[LFTLNK] if prevblock is None: # the deque has become empty; recenter instead of freeing block self.rightndx = n//2 self.leftndx = n//2+1 else: prevblock[RGTLNK] = None self.right[LFTLNK] = None self.right = prevblock self.rightndx = n-1 return x def popleft(self): if self.left is self.right and self.leftndx > self.rightndx: raise IndexError("pop from an empty deque") x = self.left[self.leftndx] self.left[self.leftndx] = None self.length -= 1 self.leftndx += 1 self.state += 1 if self.leftndx == n: prevblock = self.left[RGTLNK] if prevblock is None: # the deque has become empty; recenter instead of freeing block self.rightndx = n//2 self.leftndx = n//2+1 else: prevblock[LFTLNK] = None self.left[RGTLNK] = None self.left = prevblock self.leftndx = 0 return x def count(self, value): c = 0 for item in self: if item == value: c += 1 return c def remove(self, value): # Need to defend mutating or failing comparisons i = 0 try: for i in range(len(self)): if self[0] == value: self.popleft() return self.append(self.popleft()) i += 1 raise ValueError("deque.remove(x): x not in deque") finally: self.rotate(i) def rotate(self, n=1): length = len(self) if length <= 1: return halflen = length >> 1 if n > halflen or n < -halflen: n %= length if n > halflen: n -= length elif n < -halflen: n += length while n > 0: self.appendleft(self.pop()) n -= 1 while n < 0: self.append(self.popleft()) n += 1 def reverse(self): "reverse *IN PLACE*" leftblock = self.left rightblock = self.right leftindex = self.leftndx rightindex = self.rightndx for i in range(self.length // 2): # Validate that pointers haven't met in the middle assert leftblock != rightblock or leftindex < rightindex # Swap (rightblock[rightindex], leftblock[leftindex]) = ( leftblock[leftindex], rightblock[rightindex]) # Advance left block/index pair leftindex += 1 if leftindex == n: leftblock = leftblock[RGTLNK] assert leftblock is not None leftindex = 0 # Step backwards with the right block/index pair rightindex -= 1 if rightindex == -1: rightblock = rightblock[LFTLNK] assert rightblock is not None rightindex = n - 1 def __repr__(self): threadlocalattr = '__repr' + str(_thread_ident()) if threadlocalattr in self.__dict__: return 'deque([...])' else: self.__dict__[threadlocalattr] = True try: if self.maxlen is not None: return 'deque(%r, maxlen=%s)' % (list(self), self.maxlen) else: return 'deque(%r)' % (list(self),) finally: del self.__dict__[threadlocalattr] def __iter__(self): return deque_iterator(self, self._iter_impl) def _iter_impl(self, original_state, giveup): if self.state != original_state: giveup() block = self.left while block: l, r = 0, n if block is self.left: l = self.leftndx if block is self.right: r = self.rightndx + 1 for elem in block[l:r]: yield elem if self.state != original_state: giveup() block = block[RGTLNK] def __reversed__(self): return deque_iterator(self, self._reversed_impl) def _reversed_impl(self, original_state, giveup): if self.state != original_state: giveup() block = self.right while block: l, r = 0, n if block is self.left: l = self.leftndx if block is self.right: r = self.rightndx + 1 for elem in reversed(block[l:r]): yield elem if self.state != original_state: giveup() block = block[LFTLNK] def __len__(self): #sum = 0 #block = self.left #while block: # sum += n # block = block[RGTLNK] #return sum + self.rightndx - self.leftndx + 1 - n return self.length def __getref(self, index): if index >= 0: block = self.left while block: l, r = 0, n if block is self.left: l = self.leftndx if block is self.right: r = self.rightndx + 1 span = r-l if index < span: return block, l+index index -= span block = block[RGTLNK] else: block = self.right while block: l, r = 0, n if block is self.left: l = self.leftndx if block is self.right: r = self.rightndx + 1 negative_span = l-r if index >= negative_span: return block, r+index index -= negative_span block = block[LFTLNK] raise IndexError("deque index out of range") def __getitem__(self, index): block, index = self.__getref(index) return block[index] def __setitem__(self, index, value): block, index = self.__getref(index) block[index] = value def __delitem__(self, index): length = len(self) if index >= 0: if index >= length: raise IndexError("deque index out of range") self.rotate(-index) self.popleft() self.rotate(index) else: index = ~index if index >= length: raise IndexError("deque index out of range") self.rotate(index) self.pop() self.rotate(-index) def __reduce_ex__(self, proto): return type(self), (list(self), self.maxlen) __hash__ = None def __copy__(self): return self.__class__(self, self.maxlen) # XXX make comparison more efficient def __eq__(self, other): if isinstance(other, deque): return list(self) == list(other) else: return NotImplemented def __ne__(self, other): if isinstance(other, deque): return list(self) != list(other) else: return NotImplemented def __lt__(self, other): if isinstance(other, deque): return list(self) < list(other) else: return NotImplemented def __le__(self, other): if isinstance(other, deque): return list(self) <= list(other) else: return NotImplemented def __gt__(self, other): if isinstance(other, deque): return list(self) > list(other) else: return NotImplemented def __ge__(self, other): if isinstance(other, deque): return list(self) >= list(other) else: return NotImplemented def __iadd__(self, other): self.extend(other) return self class deque_iterator(object): def __init__(self, deq, itergen): self.counter = len(deq) def giveup(): self.counter = 0 raise RuntimeError("deque mutated during iteration") self._gen = itergen(deq.state, giveup) def next(self): res = next(self._gen) self.counter -= 1 return res def __iter__(self): return self class defaultdict(dict): def __init__(self, *args, **kwds): if len(args) > 0: default_factory = args[0] args = args[1:] if not callable(default_factory) and default_factory is not None: raise TypeError("first argument must be callable") else: default_factory = None self.default_factory = default_factory super(defaultdict, self).__init__(*args, **kwds) def __missing__(self, key): # from defaultdict docs if self.default_factory is None: raise KeyError(key) self[key] = value = self.default_factory() return value def __repr__(self, recurse=set()): if id(self) in recurse: return "defaultdict(...)" try: recurse.add(id(self)) return "defaultdict(%s, %s)" % (repr(self.default_factory), super(defaultdict, self).__repr__()) finally: recurse.remove(id(self)) def copy(self): return type(self)(self.default_factory, self) def __copy__(self): return self.copy() def __reduce__(self): """ __reduce__ must return a 5-tuple as follows: - factory function - tuple of args for the factory function - additional state (here None) - sequence iterator (here None) - dictionary iterator (yielding successive (key, value) pairs This API is used by pickle.py and copy.py. """ return (type(self), (self.default_factory,), None, None, self.iteritems()) ================================================ FILE: third_party/pypy/_csv.py ================================================ __doc__ = """CSV parsing and writing. This module provides classes that assist in the reading and writing of Comma Separated Value (CSV) files, and implements the interface described by PEP 305. Although many CSV files are simple to parse, the format is not formally defined by a stable specification and is subtle enough that parsing lines of a CSV file with something like line.split(\",\") is bound to fail. The module supports three basic APIs: reading, writing, and registration of dialects. DIALECT REGISTRATION: Readers and writers support a dialect argument, which is a convenient handle on a group of settings. When the dialect argument is a string, it identifies one of the dialects previously registered with the module. If it is a class or instance, the attributes of the argument are used as the settings for the reader or writer: class excel: delimiter = ',' quotechar = '\"' escapechar = None doublequote = True skipinitialspace = False lineterminator = '\\r\\n' quoting = QUOTE_MINIMAL SETTINGS: * quotechar - specifies a one-character string to use as the quoting character. It defaults to '\"'. * delimiter - specifies a one-character string to use as the field separator. It defaults to ','. * skipinitialspace - specifies how to interpret whitespace which immediately follows a delimiter. It defaults to False, which means that whitespace immediately following a delimiter is part of the following field. * lineterminator - specifies the character sequence which should terminate rows. * quoting - controls when quotes should be generated by the writer. It can take on any of the following module constants: csv.QUOTE_MINIMAL means only when required, for example, when a field contains either the quotechar or the delimiter csv.QUOTE_ALL means that quotes are always placed around fields. csv.QUOTE_NONNUMERIC means that quotes are always placed around fields which do not parse as integers or floating point numbers. csv.QUOTE_NONE means that quotes are never placed around fields. * escapechar - specifies a one-character string used to escape the delimiter when quoting is set to QUOTE_NONE. * doublequote - controls the handling of quotes inside fields. When True, two consecutive quotes are interpreted as one during read, and when writing, each quote character embedded in the data is written as two quotes. """ __version__ = "1.0" __all__ = [ 'Dialect', 'Error', 'QUOTE_ALL', 'QUOTE_MINIMAL', 'QUOTE_NONE', 'QUOTE_NONNUMERIC', 'Reader', 'Writer', '__doc__', '__version__', '_call_dialect', '_dialects', '_field_limit', 'field_size_limit', 'get_dialect', 'list_dialects', 'reader', 'register_dialect', 'undefined', 'unregister_dialect', 'writer' ] QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE = range(4) _dialects = {} _field_limit = 128 * 1024 # max parsed field size class Error(Exception): pass class Dialect(object): """CSV dialect The Dialect type records CSV parsing and generation options.""" __slots__ = ["_delimiter", "_doublequote", "_escapechar", "_lineterminator", "_quotechar", "_quoting", "_skipinitialspace", "_strict"] def __new__(cls, dialect, **kwargs): for name in kwargs: if '_' + name not in Dialect.__slots__: raise TypeError("unexpected keyword argument '%s'" % (name,)) if dialect is not None: if isinstance(dialect, basestring): dialect = get_dialect(dialect) # Can we reuse this instance? if (isinstance(dialect, Dialect) and all(value is None for value in kwargs.itervalues())): return dialect self = object.__new__(cls) def set_char(x): if x is None: return None if isinstance(x, str) and len(x) <= 1: return x raise TypeError("%r must be a 1-character string" % (name,)) def set_str(x): if isinstance(x, str): return x raise TypeError("%r must be a string" % (name,)) def set_quoting(x): if x in range(4): return x raise TypeError("bad 'quoting' value") attributes = {"delimiter": (',', set_char), "doublequote": (True, bool), "escapechar": (None, set_char), "lineterminator": ("\r\n", set_str), "quotechar": ('"', set_char), "quoting": (QUOTE_MINIMAL, set_quoting), "skipinitialspace": (False, bool), "strict": (False, bool), } # Copy attributes notset = object() for name in Dialect.__slots__: name = name[1:] value = notset if name in kwargs: value = kwargs[name] elif dialect is not None: value = getattr(dialect, name, notset) # mapping by name: (default, converter) if value is notset: value = attributes[name][0] if name == 'quoting' and not self.quotechar: value = QUOTE_NONE else: converter = attributes[name][1] if converter: value = converter(value) # setattr(self, '_' + name, value) self.__dict__['_' + name] = value if not self.delimiter: raise TypeError("delimiter must be set") if self.quoting != QUOTE_NONE and not self.quotechar: raise TypeError("quotechar must be set if quoting enabled") if not self.lineterminator: raise TypeError("lineterminator must be set") return self delimiter = property(lambda self: self._delimiter) doublequote = property(lambda self: self._doublequote) escapechar = property(lambda self: self._escapechar) lineterminator = property(lambda self: self._lineterminator) quotechar = property(lambda self: self._quotechar) quoting = property(lambda self: self._quoting) skipinitialspace = property(lambda self: self._skipinitialspace) strict = property(lambda self: self._strict) def _call_dialect(dialect_inst, kwargs): return Dialect(dialect_inst, **kwargs) def register_dialect(name, dialect=None, **kwargs): """Create a mapping from a string name to a dialect class. dialect = csv.register_dialect(name, dialect)""" if not isinstance(name, basestring): raise TypeError("dialect name must be a string or unicode") dialect = _call_dialect(dialect, kwargs) _dialects[name] = dialect def unregister_dialect(name): """Delete the name/dialect mapping associated with a string name.\n csv.unregister_dialect(name)""" try: del _dialects[name] except KeyError: raise Error("unknown dialect") def get_dialect(name): """Return the dialect instance associated with name. dialect = csv.get_dialect(name)""" try: return _dialects[name] except KeyError: raise Error("unknown dialect") def list_dialects(): """Return a list of all know dialect names names = csv.list_dialects()""" return list(_dialects) class Reader(object): """CSV reader Reader objects are responsible for reading and parsing tabular data in CSV format.""" (START_RECORD, START_FIELD, ESCAPED_CHAR, IN_FIELD, IN_QUOTED_FIELD, ESCAPE_IN_QUOTED_FIELD, QUOTE_IN_QUOTED_FIELD, EAT_CRNL) = range(8) def __init__(self, iterator, dialect=None, **kwargs): self.dialect = _call_dialect(dialect, kwargs) self.input_iter = iter(iterator) self.line_num = 0 self._parse_reset() def _parse_reset(self): self.field = '' self.fields = [] self.state = self.START_RECORD self.numeric_field = False def __iter__(self): return self def next(self): self._parse_reset() while True: try: line = next(self.input_iter) except StopIteration: # End of input OR exception if len(self.field) > 0: raise Error("newline inside string") raise self.line_num += 1 if '\0' in line: raise Error("line contains NULL byte") pos = 0 while pos < len(line): pos = self._parse_process_char(line, pos) self._parse_eol() if self.state == self.START_RECORD: break fields = self.fields self.fields = [] return fields def _parse_process_char(self, line, pos): c = line[pos] if self.state == self.IN_FIELD: # in unquoted field pos2 = pos while True: if c in '\n\r': # end of line - return [fields] if pos2 > pos: self._parse_add_char(line[pos:pos2]) pos = pos2 self._parse_save_field() self.state = self.EAT_CRNL elif c == self.dialect.escapechar: # possible escaped character pos2 -= 1 self.state = self.ESCAPED_CHAR elif c == self.dialect.delimiter: # save field - wait for new field if pos2 > pos: self._parse_add_char(line[pos:pos2]) pos = pos2 self._parse_save_field() self.state = self.START_FIELD else: # normal character - save in field pos2 += 1 if pos2 < len(line): c = line[pos2] continue break if pos2 > pos: self._parse_add_char(line[pos:pos2]) pos = pos2 - 1 elif self.state == self.START_RECORD: if c in '\n\r': self.state = self.EAT_CRNL else: self.state = self.START_FIELD # restart process self._parse_process_char(line, pos) elif self.state == self.START_FIELD: if c in '\n\r': # save empty field - return [fields] self._parse_save_field() self.state = self.EAT_CRNL elif (c == self.dialect.quotechar and self.dialect.quoting != QUOTE_NONE): # start quoted field self.state = self.IN_QUOTED_FIELD elif c == self.dialect.escapechar: # possible escaped character self.state = self.ESCAPED_CHAR elif c == ' ' and self.dialect.skipinitialspace: # ignore space at start of field pass elif c == self.dialect.delimiter: # save empty field self._parse_save_field() else: # begin new unquoted field if self.dialect.quoting == QUOTE_NONNUMERIC: self.numeric_field = True self._parse_add_char(c) self.state = self.IN_FIELD elif self.state == self.ESCAPED_CHAR: self._parse_add_char(c) self.state = self.IN_FIELD elif self.state == self.IN_QUOTED_FIELD: if c == self.dialect.escapechar: # possible escape character self.state = self.ESCAPE_IN_QUOTED_FIELD elif (c == self.dialect.quotechar and self.dialect.quoting != QUOTE_NONE): if self.dialect.doublequote: # doublequote; " represented by "" self.state = self.QUOTE_IN_QUOTED_FIELD else: #end of quote part of field self.state = self.IN_FIELD else: # normal character - save in field self._parse_add_char(c) elif self.state == self.ESCAPE_IN_QUOTED_FIELD: self._parse_add_char(c) self.state = self.IN_QUOTED_FIELD elif self.state == self.QUOTE_IN_QUOTED_FIELD: # doublequote - seen a quote in a quoted field if (c == self.dialect.quotechar and self.dialect.quoting != QUOTE_NONE): # save "" as " self._parse_add_char(c) self.state = self.IN_QUOTED_FIELD elif c == self.dialect.delimiter: # save field - wait for new field self._parse_save_field() self.state = self.START_FIELD elif c in '\r\n': # end of line - return [fields] self._parse_save_field() self.state = self.EAT_CRNL elif not self.dialect.strict: self._parse_add_char(c) self.state = self.IN_FIELD else: raise Error("'%c' expected after '%c'" % (self.dialect.delimiter, self.dialect.quotechar)) elif self.state == self.EAT_CRNL: if c not in '\r\n': raise Error("new-line character seen in unquoted field - " "do you need to open the file " "in universal-newline mode?") else: raise RuntimeError("unknown state: %r" % (self.state,)) return pos + 1 def _parse_eol(self): if self.state == self.EAT_CRNL: self.state = self.START_RECORD elif self.state == self.START_RECORD: # empty line - return [] pass elif self.state == self.IN_FIELD: # in unquoted field # end of line - return [fields] self._parse_save_field() self.state = self.START_RECORD elif self.state == self.START_FIELD: # save empty field - return [fields] self._parse_save_field() self.state = self.START_RECORD elif self.state == self.ESCAPED_CHAR: self._parse_add_char('\n') self.state = self.IN_FIELD elif self.state == self.IN_QUOTED_FIELD: pass elif self.state == self.ESCAPE_IN_QUOTED_FIELD: self._parse_add_char('\n') self.state = self.IN_QUOTED_FIELD elif self.state == self.QUOTE_IN_QUOTED_FIELD: # end of line - return [fields] self._parse_save_field() self.state = self.START_RECORD else: raise RuntimeError("unknown state: %r" % (self.state,)) def _parse_save_field(self): field, self.field = self.field, '' if self.numeric_field: self.numeric_field = False field = float(field) self.fields.append(field) def _parse_add_char(self, c): if len(self.field) + len(c) > _field_limit: raise Error("field larger than field limit (%d)" % (_field_limit)) self.field += c class Writer(object): """CSV writer Writer objects are responsible for generating tabular data in CSV format from sequence input.""" def __init__(self, file, dialect=None, **kwargs): if not (hasattr(file, 'write') and callable(file.write)): raise TypeError("argument 1 must have a 'write' method") self.writeline = file.write self.dialect = _call_dialect(dialect, kwargs) def _join_reset(self): self.rec = [] self.num_fields = 0 def _join_append(self, field, quoted, quote_empty): dialect = self.dialect # If this is not the first field we need a field separator if self.num_fields > 0: self.rec.append(dialect.delimiter) if dialect.quoting == QUOTE_NONE: need_escape = tuple(dialect.lineterminator) + ( dialect.escapechar, # escapechar always first dialect.delimiter, dialect.quotechar) else: for c in tuple(dialect.lineterminator) + ( dialect.delimiter, dialect.escapechar): if c and c in field: quoted = True need_escape = () if dialect.quotechar in field: if dialect.doublequote: field = field.replace(dialect.quotechar, dialect.quotechar * 2) quoted = True else: need_escape = (dialect.quotechar,) for c in need_escape: if c and c in field: if not dialect.escapechar: raise Error("need to escape, but no escapechar set") field = field.replace(c, dialect.escapechar + c) # If field is empty check if it needs to be quoted if field == '' and quote_empty: if dialect.quoting == QUOTE_NONE: raise Error("single empty field record must be quoted") quoted = 1 if quoted: field = dialect.quotechar + field + dialect.quotechar self.rec.append(field) self.num_fields += 1 def writerow(self, row): dialect = self.dialect try: rowlen = len(row) except TypeError: raise Error("sequence expected") # join all fields in internal buffer self._join_reset() for field in row: quoted = False if dialect.quoting == QUOTE_NONNUMERIC: try: float(field) except: quoted = True # This changed since 2.5: # quoted = not isinstance(field, (int, long, float)) elif dialect.quoting == QUOTE_ALL: quoted = True if field is None: value = "" elif isinstance(field, float): value = repr(field) else: value = str(field) self._join_append(value, quoted, rowlen == 1) # add line terminator self.rec.append(dialect.lineterminator) self.writeline(''.join(self.rec)) def writerows(self, rows): for row in rows: self.writerow(row) def reader(*args, **kwargs): """ csv_reader = reader(iterable [, dialect='excel'] [optional keyword args]) for row in csv_reader: process(row) The "iterable" argument can be any object that returns a line of input for each iteration, such as a file object or a list. The optional \"dialect\" parameter is discussed below. The function also accepts optional keyword arguments which override settings provided by the dialect. The returned object is an iterator. Each iteration returns a row of the CSV file (which can span multiple input lines)""" return Reader(*args, **kwargs) def writer(*args, **kwargs): """ csv_writer = csv.writer(fileobj [, dialect='excel'] [optional keyword args]) for row in sequence: csv_writer.writerow(row) [or] csv_writer = csv.writer(fileobj [, dialect='excel'] [optional keyword args]) csv_writer.writerows(rows) The \"fileobj\" argument can be any object that supports the file API.""" return Writer(*args, **kwargs) undefined = object() def field_size_limit(limit=undefined): """Sets an upper limit on parsed fields. csv.field_size_limit([limit]) Returns old limit. If limit is not given, no new limit is set and the old limit is returned""" global _field_limit old_limit = _field_limit if limit is not undefined: if not isinstance(limit, (int, long)): raise TypeError("int expected, got %s" % (limit.__class__.__name__,)) _field_limit = limit return old_limit ================================================ FILE: third_party/pypy/_functools.py ================================================ """ Supplies the internal functions for functools.py in the standard library """ __all__ = ['reduce', 'partial'] # reduce() has moved to _functools in Python 2.6+. def reduce(function, iterable, initializer=None): it = iter(iterable) if initializer is None: try: initializer = next(it) except StopIteration: raise TypeError('reduce() of empty sequence with no initial value') accum_value = initializer for x in it: accum_value = function(accum_value, x) return accum_value class partial(object): """ partial(func, *args, **keywords) - new function with partial application of the given arguments and keywords. """ __slots__ = ('_func', '_args', '_keywords', '__dict__') def __init__(*args, **keywords): if len(args) < 2: raise TypeError('__init__() takes at least 2 arguments (%d given)' % len(args)) self, func, args = args[0], args[1], args[2:] if not callable(func): raise TypeError("the first argument must be callable") self._func = func self._args = args self._keywords = keywords def __delattr__(self, key): if key == '__dict__': raise TypeError("a partial object's dictionary may not be deleted") object.__delattr__(self, key) # @property def func(self): return self._func # TODO: Make this a decorator once they're implemented. func = property(func) # @property def args(self): return self._args # TODO: Make this a decorator once they're implemented. args = property(args) # @property def keywords(self): return self._keywords # TODO: Make this a decorator once they're implemented. keywords = property(keywords) def __call__(self, *fargs, **fkeywords): if self._keywords: fkeywords = dict(self._keywords, **fkeywords) return self._func(*(self._args + fargs), **fkeywords) def __reduce__(self): d = dict((k, v) for k, v in self.__dict__.iteritems() if k not in ('_func', '_args', '_keywords')) if len(d) == 0: d = None return (type(self), (self._func,), (self._func, self._args, self._keywords, d)) def __setstate__(self, state): if not isinstance(state, tuple) or len(state) != 4: raise TypeError("invalid partial state") func, args, keywords, d = state if (not callable(func) or not isinstance(args, tuple) or (keywords is not None and not isinstance(keywords, dict))): raise TypeError("invalid partial state") self._func = func self._args = tuple(args) if keywords is None: keywords = {} elif type(keywords) is not dict: keywords = dict(keywords) self._keywords = keywords if d is None: self.__dict__.clear() else: self.__dict__.update(d) ================================================ FILE: third_party/pypy/_md5.py ================================================ #!/usr/bin/env python # -*- coding: iso-8859-1 -*- # Note that PyPy contains also a built-in module 'md5' which will hide # this one if compiled in. """A sample implementation of MD5 in pure Python. This is an implementation of the MD5 hash function, as specified by RFC 1321, in pure Python. It was implemented using Bruce Schneier's excellent book "Applied Cryptography", 2nd ed., 1996. Surely this is not meant to compete with the existing implementation of the Python standard library (written in C). Rather, it should be seen as a Python complement that is more readable than C and can be used more conveniently for learning and experimenting purposes in the field of cryptography. This module tries very hard to follow the API of the existing Python standard library's "md5" module, but although it seems to work fine, it has not been extensively tested! (But note that there is a test module, test_md5py.py, that compares this Python implementation with the C one of the Python standard library. BEWARE: this comes with no guarantee whatsoever about fitness and/or other properties! Specifically, do not use this in any production code! License is Python License! Special thanks to Aurelian Coman who fixed some nasty bugs! Dinu C. Gherman """ __date__ = '2004-11-17' __version__ = 0.91 # Modernised by J. Hall�n and L. Creighton for Pypy __metaclass__ = type # or genrpy won't work import _struct as struct import copy # ====================================================================== # Bit-Manipulation helpers # ====================================================================== def _bytelist2long(list): "Transform a list of characters into a list of longs." imax = len(list) // 4 hl = [0] * imax j = 0 i = 0 while i < imax: b0 = ord(list[j]) b1 = ord(list[j + 1]) << 8 b2 = ord(list[j + 2]) << 16 b3 = ord(list[j + 3]) << 24 hl[i] = b0 | b1 | b2 | b3 i = i + 1 j = j + 4 return hl def _rotateLeft(x, n): "Rotate x (32 bit) left n bits circularly." return (x << n) | (x >> (32 - n)) # ====================================================================== # The real MD5 meat... # # Implemented after "Applied Cryptography", 2nd ed., 1996, # pp. 436-441 by Bruce Schneier. # ====================================================================== # F, G, H and I are basic MD5 functions. def F(x, y, z): return (x & y) | ((~x) & z) def G(x, y, z): return (x & z) | (y & (~z)) def H(x, y, z): return x ^ y ^ z def I(x, y, z): return y ^ (x | (~z)) def XX(func, a, b, c, d, x, s, ac): """Wrapper for call distribution to functions F, G, H and I. This replaces functions FF, GG, HH and II from "Appl. Crypto." Rotation is separate from addition to prevent recomputation (now summed-up in one function). """ res = 0 res = res + a + func(b, c, d) res = res + x res = res + ac res = res & 0xffffffff res = _rotateLeft(res, s) res = res & 0xffffffff res = res + b return res & 0xffffffff class MD5Type(object): "An implementation of the MD5 hash function in pure Python." digest_size = digestsize = 16 block_size = 64 def __init__(self): "Initialisation." # Initial message length in bits(!). self.length = 0 self.count = [0, 0] # Initial empty message as a sequence of bytes (8 bit characters). self.input = [] # Call a separate init function, that can be used repeatedly # to start from scratch on the same object. self.init() def init(self): "Initialize the message-digest and set all fields to zero.code" self.length = 0 self.count = [0, 0] self.input = [] # Load magic initialization constants. self.A = 0x67452301 self.B = 0xefcdab89 self.C = 0x98badcfe self.D = 0x10325476 def _transform(self, inp): """Basic MD5 step transforming the digest based on the input. Note that if the Mysterious Constants are arranged backwards in little-endian order and decrypted with the DES they produce OCCULT MESSAGES! """ a, b, c, d = A, B, C, D = self.A, self.B, self.C, self.D # Round 1. S11, S12, S13, S14 = 7, 12, 17, 22 a = XX(F, a, b, c, d, inp[0], S11, 0xD76AA478) # 1 d = XX(F, d, a, b, c, inp[1], S12, 0xE8C7B756) # 2 c = XX(F, c, d, a, b, inp[2], S13, 0x242070DB) # 3 b = XX(F, b, c, d, a, inp[3], S14, 0xC1BDCEEE) # 4 a = XX(F, a, b, c, d, inp[4], S11, 0xF57C0FAF) # 5 d = XX(F, d, a, b, c, inp[5], S12, 0x4787C62A) # 6 c = XX(F, c, d, a, b, inp[6], S13, 0xA8304613) # 7 b = XX(F, b, c, d, a, inp[7], S14, 0xFD469501) # 8 a = XX(F, a, b, c, d, inp[8], S11, 0x698098D8) # 9 d = XX(F, d, a, b, c, inp[9], S12, 0x8B44F7AF) # 10 c = XX(F, c, d, a, b, inp[10], S13, 0xFFFF5BB1) # 11 b = XX(F, b, c, d, a, inp[11], S14, 0x895CD7BE) # 12 a = XX(F, a, b, c, d, inp[12], S11, 0x6B901122) # 13 d = XX(F, d, a, b, c, inp[13], S12, 0xFD987193) # 14 c = XX(F, c, d, a, b, inp[14], S13, 0xA679438E) # 15 b = XX(F, b, c, d, a, inp[15], S14, 0x49B40821) # 16 # Round 2. S21, S22, S23, S24 = 5, 9, 14, 20 a = XX(G, a, b, c, d, inp[1], S21, 0xF61E2562) # 17 d = XX(G, d, a, b, c, inp[6], S22, 0xC040B340) # 18 c = XX(G, c, d, a, b, inp[11], S23, 0x265E5A51) # 19 b = XX(G, b, c, d, a, inp[0], S24, 0xE9B6C7AA) # 20 a = XX(G, a, b, c, d, inp[5], S21, 0xD62F105D) # 21 d = XX(G, d, a, b, c, inp[10], S22, 0x02441453) # 22 c = XX(G, c, d, a, b, inp[15], S23, 0xD8A1E681) # 23 b = XX(G, b, c, d, a, inp[4], S24, 0xE7D3FBC8) # 24 a = XX(G, a, b, c, d, inp[9], S21, 0x21E1CDE6) # 25 d = XX(G, d, a, b, c, inp[14], S22, 0xC33707D6) # 26 c = XX(G, c, d, a, b, inp[3], S23, 0xF4D50D87) # 27 b = XX(G, b, c, d, a, inp[8], S24, 0x455A14ED) # 28 a = XX(G, a, b, c, d, inp[13], S21, 0xA9E3E905) # 29 d = XX(G, d, a, b, c, inp[2], S22, 0xFCEFA3F8) # 30 c = XX(G, c, d, a, b, inp[7], S23, 0x676F02D9) # 31 b = XX(G, b, c, d, a, inp[12], S24, 0x8D2A4C8A) # 32 # Round 3. S31, S32, S33, S34 = 4, 11, 16, 23 a = XX(H, a, b, c, d, inp[5], S31, 0xFFFA3942) # 33 d = XX(H, d, a, b, c, inp[8], S32, 0x8771F681) # 34 c = XX(H, c, d, a, b, inp[11], S33, 0x6D9D6122) # 35 b = XX(H, b, c, d, a, inp[14], S34, 0xFDE5380C) # 36 a = XX(H, a, b, c, d, inp[1], S31, 0xA4BEEA44) # 37 d = XX(H, d, a, b, c, inp[4], S32, 0x4BDECFA9) # 38 c = XX(H, c, d, a, b, inp[7], S33, 0xF6BB4B60) # 39 b = XX(H, b, c, d, a, inp[10], S34, 0xBEBFBC70) # 40 a = XX(H, a, b, c, d, inp[13], S31, 0x289B7EC6) # 41 d = XX(H, d, a, b, c, inp[0], S32, 0xEAA127FA) # 42 c = XX(H, c, d, a, b, inp[3], S33, 0xD4EF3085) # 43 b = XX(H, b, c, d, a, inp[6], S34, 0x04881D05) # 44 a = XX(H, a, b, c, d, inp[9], S31, 0xD9D4D039) # 45 d = XX(H, d, a, b, c, inp[12], S32, 0xE6DB99E5) # 46 c = XX(H, c, d, a, b, inp[15], S33, 0x1FA27CF8) # 47 b = XX(H, b, c, d, a, inp[2], S34, 0xC4AC5665) # 48 # Round 4. S41, S42, S43, S44 = 6, 10, 15, 21 a = XX(I, a, b, c, d, inp[0], S41, 0xF4292244) # 49 d = XX(I, d, a, b, c, inp[7], S42, 0x432AFF97) # 50 c = XX(I, c, d, a, b, inp[14], S43, 0xAB9423A7) # 51 b = XX(I, b, c, d, a, inp[5], S44, 0xFC93A039) # 52 a = XX(I, a, b, c, d, inp[12], S41, 0x655B59C3) # 53 d = XX(I, d, a, b, c, inp[3], S42, 0x8F0CCC92) # 54 c = XX(I, c, d, a, b, inp[10], S43, 0xFFEFF47D) # 55 b = XX(I, b, c, d, a, inp[1], S44, 0x85845DD1) # 56 a = XX(I, a, b, c, d, inp[8], S41, 0x6FA87E4F) # 57 d = XX(I, d, a, b, c, inp[15], S42, 0xFE2CE6E0) # 58 c = XX(I, c, d, a, b, inp[6], S43, 0xA3014314) # 59 b = XX(I, b, c, d, a, inp[13], S44, 0x4E0811A1) # 60 a = XX(I, a, b, c, d, inp[4], S41, 0xF7537E82) # 61 d = XX(I, d, a, b, c, inp[11], S42, 0xBD3AF235) # 62 c = XX(I, c, d, a, b, inp[2], S43, 0x2AD7D2BB) # 63 b = XX(I, b, c, d, a, inp[9], S44, 0xEB86D391) # 64 A = (A + a) & 0xffffffff B = (B + b) & 0xffffffff C = (C + c) & 0xffffffff D = (D + d) & 0xffffffff self.A, self.B, self.C, self.D = A, B, C, D # Down from here all methods follow the Python Standard Library # API of the md5 module. def update(self, inBuf): """Add to the current message. Update the md5 object with the string arg. Repeated calls are equivalent to a single call with the concatenation of all the arguments, i.e. m.update(a); m.update(b) is equivalent to m.update(a+b). The hash is immediately calculated for all full blocks. The final calculation is made in digest(). This allows us to keep an intermediate value for the hash, so that we only need to make minimal recalculation if we call update() to add moredata to the hashed string. """ leninBuf = len(inBuf) # Compute number of bytes mod 64. index = (self.count[0] >> 3) & 0x3F # Update number of bits. self.count[0] = self.count[0] + (leninBuf << 3) if self.count[0] < (leninBuf << 3): self.count[1] = self.count[1] + 1 self.count[1] = self.count[1] + (leninBuf >> 29) partLen = 64 - index if leninBuf >= partLen: self.input[index:] = list(inBuf[:partLen]) self._transform(_bytelist2long(self.input)) i = partLen while i + 63 < leninBuf: self._transform(_bytelist2long(list(inBuf[i:i + 64]))) i = i + 64 else: self.input = list(inBuf[i:leninBuf]) else: i = 0 self.input = self.input + list(inBuf) def digest(self): """Terminate the message-digest computation and return digest. Return the digest of the strings passed to the update() method so far. This is a 16-byte string which may contain non-ASCII characters, including null bytes. """ A = self.A B = self.B C = self.C D = self.D input = [] + self.input count = [] + self.count index = (self.count[0] >> 3) & 0x3f if index < 56: padLen = 56 - index else: padLen = 120 - index padding = [b'\200'] + [b'\000'] * 63 self.update(padding[:padLen]) # Append length (before padding). bits = _bytelist2long(self.input[:56]) + count self._transform(bits) # Store state in digest. digest = struct.pack(" 0: s = pack('>I', n & 0xffffffff) + s n = n >> 32 # Strip off leading zeros. for i in range(len(s)): if s[i] != '\000': break else: # Only happens when n == 0. s = '\000' i = 0 s = s[i:] # Add back some pad bytes. This could be done more efficiently # w.r.t. the de-padding being done above, but sigh... if blocksize > 0 and len(s) % blocksize: s = (blocksize - len(s) % blocksize) * '\000' + s return s def _bytelist2longBigEndian(list): "Transform a list of characters into a list of longs." imax = len(list) // 4 hl = [0] * imax j = 0 i = 0 while i < imax: b0 = ord(list[j]) << 24 b1 = ord(list[j+1]) << 16 b2 = ord(list[j+2]) << 8 b3 = ord(list[j+3]) hl[i] = b0 | b1 | b2 | b3 i = i+1 j = j+4 return hl def _rotateLeft(x, n): "Rotate x (32 bit) left n bits circularly." return (x << n) | (x >> (32-n)) # ====================================================================== # The SHA transformation functions # # ====================================================================== def f0_19(B, C, D): return (B & C) | ((~ B) & D) def f20_39(B, C, D): return B ^ C ^ D def f40_59(B, C, D): return (B & C) | (B & D) | (C & D) def f60_79(B, C, D): return B ^ C ^ D f = [f0_19, f20_39, f40_59, f60_79] # Constants to be used K = [ 0x5A827999, # ( 0 <= t <= 19) 0x6ED9EBA1, # (20 <= t <= 39) 0x8F1BBCDC, # (40 <= t <= 59) 0xCA62C1D6 # (60 <= t <= 79) ] class sha(object): "An implementation of the SHA hash function in pure Python." digest_size = digestsize = 20 block_size = 512 // 8 def __init__(self): "Initialisation." # Initial message length in bits(!). self.length = 0 self.count = [0, 0] # Initial empty message as a sequence of bytes (8 bit characters). self.input = [] # Call a separate init function, that can be used repeatedly # to start from scratch on the same object. self.init() def init(self): "Initialize the message-digest and set all fields to zero." self.length = 0 self.input = [] # Initial 160 bit message digest (5 times 32 bit). self.H0 = 0x67452301 self.H1 = 0xEFCDAB89 self.H2 = 0x98BADCFE self.H3 = 0x10325476 self.H4 = 0xC3D2E1F0 def _transform(self, W): for t in range(16, 80): W.append(_rotateLeft( W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16], 1) & 0xffffffff) A = self.H0 B = self.H1 C = self.H2 D = self.H3 E = self.H4 """ This loop was unrolled to gain about 10% in speed for t in range(0, 80): TEMP = _rotateLeft(A, 5) + f[t/20] + E + W[t] + K[t/20] E = D D = C C = _rotateLeft(B, 30) & 0xffffffff B = A A = TEMP & 0xffffffff """ for t in range(0, 20): TEMP = _rotateLeft(A, 5) + ((B & C) | ((~ B) & D)) + E + W[t] + K[0] E = D D = C C = _rotateLeft(B, 30) & 0xffffffff B = A A = TEMP & 0xffffffff for t in range(20, 40): TEMP = _rotateLeft(A, 5) + (B ^ C ^ D) + E + W[t] + K[1] E = D D = C C = _rotateLeft(B, 30) & 0xffffffff B = A A = TEMP & 0xffffffff for t in range(40, 60): TEMP = _rotateLeft(A, 5) + ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2] E = D D = C C = _rotateLeft(B, 30) & 0xffffffff B = A A = TEMP & 0xffffffff for t in range(60, 80): TEMP = _rotateLeft(A, 5) + (B ^ C ^ D) + E + W[t] + K[3] E = D D = C C = _rotateLeft(B, 30) & 0xffffffff B = A A = TEMP & 0xffffffff self.H0 = (self.H0 + A) & 0xffffffff self.H1 = (self.H1 + B) & 0xffffffff self.H2 = (self.H2 + C) & 0xffffffff self.H3 = (self.H3 + D) & 0xffffffff self.H4 = (self.H4 + E) & 0xffffffff # Down from here all methods follow the Python Standard Library # API of the sha module. def update(self, inBuf): """Add to the current message. Update the md5 object with the string arg. Repeated calls are equivalent to a single call with the concatenation of all the arguments, i.e. m.update(a); m.update(b) is equivalent to m.update(a+b). The hash is immediately calculated for all full blocks. The final calculation is made in digest(). It will calculate 1-2 blocks, depending on how much padding we have to add. This allows us to keep an intermediate value for the hash, so that we only need to make minimal recalculation if we call update() to add more data to the hashed string. """ leninBuf = len(inBuf) # Compute number of bytes mod 64. index = (self.count[1] >> 3) & 0x3F # Update number of bits. self.count[1] = self.count[1] + (leninBuf << 3) if self.count[1] < (leninBuf << 3): self.count[0] = self.count[0] + 1 self.count[0] = self.count[0] + (leninBuf >> 29) partLen = 64 - index if leninBuf >= partLen: self.input[index:] = list(inBuf[:partLen]) self._transform(_bytelist2longBigEndian(self.input)) i = partLen while i + 63 < leninBuf: self._transform(_bytelist2longBigEndian(list(inBuf[i:i+64]))) i = i + 64 else: self.input = list(inBuf[i:leninBuf]) else: i = 0 self.input = self.input + list(inBuf) def digest(self): """Terminate the message-digest computation and return digest. Return the digest of the strings passed to the update() method so far. This is a 16-byte string which may contain non-ASCII characters, including null bytes. """ H0 = self.H0 H1 = self.H1 H2 = self.H2 H3 = self.H3 H4 = self.H4 input = [] + self.input count = [] + self.count index = (self.count[1] >> 3) & 0x3f if index < 56: padLen = 56 - index else: padLen = 120 - index padding = ['\200'] + ['\000'] * 63 self.update(padding[:padLen]) # Append length (before padding). bits = _bytelist2longBigEndian(self.input[:56]) + count self._transform(bits) # Store state in digest. digest = _long2bytesBigEndian(self.H0, 4) + \ _long2bytesBigEndian(self.H1, 4) + \ _long2bytesBigEndian(self.H2, 4) + \ _long2bytesBigEndian(self.H3, 4) + \ _long2bytesBigEndian(self.H4, 4) self.H0 = H0 self.H1 = H1 self.H2 = H2 self.H3 = H3 self.H4 = H4 self.input = input self.count = count return digest def hexdigest(self): """Terminate and return digest in HEX form. Like digest() except the digest is returned as a string of length 32, containing only hexadecimal digits. This may be used to exchange the value safely in email or other non- binary environments. """ return ''.join([('0%x' % ord(c))[-2:] for c in self.digest()]) def copy(self): """Return a clone object. Return a copy ('clone') of the md5 object. This can be used to efficiently compute the digests of strings that share a common initial substring. """ return copy.deepcopy(self) # ====================================================================== # Mimic Python top-level functions from standard library API # for consistency with the _sha module of the standard library. # ====================================================================== # These are mandatory variables in the module. They have constant values # in the SHA standard. digest_size = 20 digestsize = 20 blocksize = 1 def new(arg=None): """Return a new sha crypto object. If arg is present, the method call update(arg) is made. """ crypto = sha() if arg: crypto.update(arg) return crypto ================================================ FILE: third_party/pypy/_sha256.py ================================================ import _struct as struct SHA_BLOCKSIZE = 64 SHA_DIGESTSIZE = 32 def new_shaobject(): return { 'digest': [0]*8, 'count_lo': 0, 'count_hi': 0, 'data': [0]* SHA_BLOCKSIZE, 'local': 0, 'digestsize': 0 } ROR = lambda x, y: (((x & 0xffffffff) >> (y & 31)) | (x << (32 - (y & 31)))) & 0xffffffff Ch = lambda x, y, z: (z ^ (x & (y ^ z))) Maj = lambda x, y, z: (((x | y) & z) | (x & y)) S = lambda x, n: ROR(x, n) R = lambda x, n: (x & 0xffffffff) >> n Sigma0 = lambda x: (S(x, 2) ^ S(x, 13) ^ S(x, 22)) Sigma1 = lambda x: (S(x, 6) ^ S(x, 11) ^ S(x, 25)) Gamma0 = lambda x: (S(x, 7) ^ S(x, 18) ^ R(x, 3)) Gamma1 = lambda x: (S(x, 17) ^ S(x, 19) ^ R(x, 10)) def sha_transform(sha_info): W = [] d = sha_info['data'] for i in xrange(0,16): W.append( (d[4*i]<<24) + (d[4*i+1]<<16) + (d[4*i+2]<<8) + d[4*i+3]) for i in xrange(16,64): W.append( (Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xffffffff ) ss = sha_info['digest'][:] def RND(a,b,c,d,e,f,g,h,i,ki): t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; t1 = Sigma0(a) + Maj(a, b, c); d += t0; h = t0 + t1; return d & 0xffffffff, h & 0xffffffff ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],0,0x428a2f98); ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],1,0x71374491); ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],2,0xb5c0fbcf); ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],3,0xe9b5dba5); ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],4,0x3956c25b); ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],5,0x59f111f1); ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],6,0x923f82a4); ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],7,0xab1c5ed5); ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],8,0xd807aa98); ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],9,0x12835b01); ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],10,0x243185be); ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],11,0x550c7dc3); ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],12,0x72be5d74); ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],13,0x80deb1fe); ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],14,0x9bdc06a7); ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],15,0xc19bf174); ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],16,0xe49b69c1); ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],17,0xefbe4786); ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],18,0x0fc19dc6); ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],19,0x240ca1cc); ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],20,0x2de92c6f); ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],21,0x4a7484aa); ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],22,0x5cb0a9dc); ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],23,0x76f988da); ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],24,0x983e5152); ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],25,0xa831c66d); ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],26,0xb00327c8); ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],27,0xbf597fc7); ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],28,0xc6e00bf3); ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],29,0xd5a79147); ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],30,0x06ca6351); ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],31,0x14292967); ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],32,0x27b70a85); ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],33,0x2e1b2138); ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],34,0x4d2c6dfc); ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],35,0x53380d13); ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],36,0x650a7354); ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],37,0x766a0abb); ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],38,0x81c2c92e); ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],39,0x92722c85); ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],40,0xa2bfe8a1); ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],41,0xa81a664b); ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],42,0xc24b8b70); ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],43,0xc76c51a3); ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],44,0xd192e819); ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],45,0xd6990624); ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],46,0xf40e3585); ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],47,0x106aa070); ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],48,0x19a4c116); ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],49,0x1e376c08); ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],50,0x2748774c); ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],51,0x34b0bcb5); ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],52,0x391c0cb3); ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],53,0x4ed8aa4a); ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],54,0x5b9cca4f); ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],55,0x682e6ff3); ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],56,0x748f82ee); ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],57,0x78a5636f); ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],58,0x84c87814); ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],59,0x8cc70208); ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],60,0x90befffa); ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],61,0xa4506ceb); ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],62,0xbef9a3f7); ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],63,0xc67178f2); dig = [] for i, x in enumerate(sha_info['digest']): dig.append( (x + ss[i]) & 0xffffffff ) sha_info['digest'] = dig def sha_init(): sha_info = new_shaobject() sha_info['digest'] = [0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19] sha_info['count_lo'] = 0 sha_info['count_hi'] = 0 sha_info['local'] = 0 sha_info['digestsize'] = 32 return sha_info def sha224_init(): sha_info = new_shaobject() sha_info['digest'] = [0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4] sha_info['count_lo'] = 0 sha_info['count_hi'] = 0 sha_info['local'] = 0 sha_info['digestsize'] = 28 return sha_info def getbuf(s): if isinstance(s, str): return s elif isinstance(s, unicode): return str(s) else: return buffer(s) def sha_update(sha_info, buffer): count = len(buffer) buffer_idx = 0 clo = (sha_info['count_lo'] + (count << 3)) & 0xffffffff if clo < sha_info['count_lo']: sha_info['count_hi'] += 1 sha_info['count_lo'] = clo sha_info['count_hi'] += (count >> 29) if sha_info['local']: i = SHA_BLOCKSIZE - sha_info['local'] if i > count: i = count # copy buffer for x in enumerate(buffer[buffer_idx:buffer_idx+i]): sha_info['data'][sha_info['local']+x[0]] = struct.unpack('B', x[1])[0] count -= i buffer_idx += i sha_info['local'] += i if sha_info['local'] == SHA_BLOCKSIZE: sha_transform(sha_info) sha_info['local'] = 0 else: return while count >= SHA_BLOCKSIZE: # copy buffer sha_info['data'] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + SHA_BLOCKSIZE]] count -= SHA_BLOCKSIZE buffer_idx += SHA_BLOCKSIZE sha_transform(sha_info) # copy buffer pos = sha_info['local'] sha_info['data'][pos:pos+count] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + count]] sha_info['local'] = count def sha_final(sha_info): lo_bit_count = sha_info['count_lo'] hi_bit_count = sha_info['count_hi'] count = (lo_bit_count >> 3) & 0x3f sha_info['data'][count] = 0x80; count += 1 if count > SHA_BLOCKSIZE - 8: # zero the bytes in data after the count sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) sha_transform(sha_info) # zero bytes in data sha_info['data'] = [0] * SHA_BLOCKSIZE else: sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) sha_info['data'][56] = (hi_bit_count >> 24) & 0xff sha_info['data'][57] = (hi_bit_count >> 16) & 0xff sha_info['data'][58] = (hi_bit_count >> 8) & 0xff sha_info['data'][59] = (hi_bit_count >> 0) & 0xff sha_info['data'][60] = (lo_bit_count >> 24) & 0xff sha_info['data'][61] = (lo_bit_count >> 16) & 0xff sha_info['data'][62] = (lo_bit_count >> 8) & 0xff sha_info['data'][63] = (lo_bit_count >> 0) & 0xff sha_transform(sha_info) dig = [] for i in sha_info['digest']: dig.extend([ ((i>>24) & 0xff), ((i>>16) & 0xff), ((i>>8) & 0xff), (i & 0xff) ]) return ''.join([chr(i) for i in dig]) class sha256(object): digest_size = digestsize = SHA_DIGESTSIZE block_size = SHA_BLOCKSIZE def __init__(self, s=None): self._sha = sha_init() if s: sha_update(self._sha, getbuf(s)) def update(self, s): sha_update(self._sha, getbuf(s)) def digest(self): return sha_final(self._sha.copy())[:self._sha['digestsize']] def hexdigest(self): return ''.join([('0%x' % ord(i))[-2:] for i in self.digest()]) def copy(self): new = sha256.__new__(sha256) new._sha = self._sha.copy() return new class sha224(sha256): digest_size = digestsize = 28 def __init__(self, s=None): self._sha = sha224_init() if s: sha_update(self._sha, getbuf(s)) def copy(self): new = sha224.__new__(sha224) new._sha = self._sha.copy() return new def test(): a_str = "just a test string" assert 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' == sha256().hexdigest() assert 'd7b553c6f09ac85d142415f857c5310f3bbbe7cdd787cce4b985acedd585266f' == sha256(a_str).hexdigest() assert '8113ebf33c97daa9998762aacafe750c7cefc2b2f173c90c59663a57fe626f21' == sha256(a_str*7).hexdigest() s = sha256(a_str) s.update(a_str) assert '03d9963e05a094593190b6fc794cb1a3e1ac7d7883f0b5855268afeccc70d461' == s.hexdigest() if __name__ == "__main__": test() ================================================ FILE: third_party/pypy/_sha512.py ================================================ """ This code was Ported from CPython's sha512module.c """ import _struct as struct SHA_BLOCKSIZE = 128 SHA_DIGESTSIZE = 64 def new_shaobject(): return { 'digest': [0]*8, 'count_lo': 0, 'count_hi': 0, 'data': [0]* SHA_BLOCKSIZE, 'local': 0, 'digestsize': 0 } ROR64 = lambda x, y: (((x & 0xffffffffffffffff) >> (y & 63)) | (x << (64 - (y & 63)))) & 0xffffffffffffffff Ch = lambda x, y, z: (z ^ (x & (y ^ z))) Maj = lambda x, y, z: (((x | y) & z) | (x & y)) S = lambda x, n: ROR64(x, n) R = lambda x, n: (x & 0xffffffffffffffff) >> n Sigma0 = lambda x: (S(x, 28) ^ S(x, 34) ^ S(x, 39)) Sigma1 = lambda x: (S(x, 14) ^ S(x, 18) ^ S(x, 41)) Gamma0 = lambda x: (S(x, 1) ^ S(x, 8) ^ R(x, 7)) Gamma1 = lambda x: (S(x, 19) ^ S(x, 61) ^ R(x, 6)) def sha_transform(sha_info): W = [] d = sha_info['data'] for i in xrange(0,16): W.append( (d[8*i]<<56) + (d[8*i+1]<<48) + (d[8*i+2]<<40) + (d[8*i+3]<<32) + (d[8*i+4]<<24) + (d[8*i+5]<<16) + (d[8*i+6]<<8) + d[8*i+7]) for i in xrange(16,80): W.append( (Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xffffffffffffffff ) ss = sha_info['digest'][:] def RND(a,b,c,d,e,f,g,h,i,ki): t0 = (h + Sigma1(e) + Ch(e, f, g) + ki + W[i]) & 0xffffffffffffffff t1 = (Sigma0(a) + Maj(a, b, c)) & 0xffffffffffffffff d = (d + t0) & 0xffffffffffffffff h = (t0 + t1) & 0xffffffffffffffff return d & 0xffffffffffffffff, h & 0xffffffffffffffff ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],0,0x428a2f98d728ae22) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],1,0x7137449123ef65cd) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],2,0xb5c0fbcfec4d3b2f) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],3,0xe9b5dba58189dbbc) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],4,0x3956c25bf348b538) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],5,0x59f111f1b605d019) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],6,0x923f82a4af194f9b) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],7,0xab1c5ed5da6d8118) ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],8,0xd807aa98a3030242) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],9,0x12835b0145706fbe) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],10,0x243185be4ee4b28c) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],11,0x550c7dc3d5ffb4e2) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],12,0x72be5d74f27b896f) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],13,0x80deb1fe3b1696b1) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],14,0x9bdc06a725c71235) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],15,0xc19bf174cf692694) ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],16,0xe49b69c19ef14ad2) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],17,0xefbe4786384f25e3) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],18,0x0fc19dc68b8cd5b5) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],19,0x240ca1cc77ac9c65) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],20,0x2de92c6f592b0275) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],21,0x4a7484aa6ea6e483) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],22,0x5cb0a9dcbd41fbd4) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],23,0x76f988da831153b5) ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],24,0x983e5152ee66dfab) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],25,0xa831c66d2db43210) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],26,0xb00327c898fb213f) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],27,0xbf597fc7beef0ee4) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],28,0xc6e00bf33da88fc2) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],29,0xd5a79147930aa725) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],30,0x06ca6351e003826f) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],31,0x142929670a0e6e70) ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],32,0x27b70a8546d22ffc) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],33,0x2e1b21385c26c926) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],34,0x4d2c6dfc5ac42aed) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],35,0x53380d139d95b3df) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],36,0x650a73548baf63de) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],37,0x766a0abb3c77b2a8) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],38,0x81c2c92e47edaee6) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],39,0x92722c851482353b) ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],40,0xa2bfe8a14cf10364) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],41,0xa81a664bbc423001) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],42,0xc24b8b70d0f89791) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],43,0xc76c51a30654be30) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],44,0xd192e819d6ef5218) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],45,0xd69906245565a910) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],46,0xf40e35855771202a) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],47,0x106aa07032bbd1b8) ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],48,0x19a4c116b8d2d0c8) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],49,0x1e376c085141ab53) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],50,0x2748774cdf8eeb99) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],51,0x34b0bcb5e19b48a8) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],52,0x391c0cb3c5c95a63) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],53,0x4ed8aa4ae3418acb) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],54,0x5b9cca4f7763e373) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],55,0x682e6ff3d6b2b8a3) ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],56,0x748f82ee5defb2fc) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],57,0x78a5636f43172f60) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],58,0x84c87814a1f0ab72) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],59,0x8cc702081a6439ec) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],60,0x90befffa23631e28) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],61,0xa4506cebde82bde9) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],62,0xbef9a3f7b2c67915) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],63,0xc67178f2e372532b) ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],64,0xca273eceea26619c) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],65,0xd186b8c721c0c207) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],66,0xeada7dd6cde0eb1e) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],67,0xf57d4f7fee6ed178) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],68,0x06f067aa72176fba) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],69,0x0a637dc5a2c898a6) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],70,0x113f9804bef90dae) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],71,0x1b710b35131c471b) ss[3], ss[7] = RND(ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],72,0x28db77f523047d84) ss[2], ss[6] = RND(ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],73,0x32caab7b40c72493) ss[1], ss[5] = RND(ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],ss[5],74,0x3c9ebe0a15c9bebc) ss[0], ss[4] = RND(ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],ss[4],75,0x431d67c49c100d4c) ss[7], ss[3] = RND(ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],ss[3],76,0x4cc5d4becb3e42b6) ss[6], ss[2] = RND(ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],ss[2],77,0x597f299cfc657e2a) ss[5], ss[1] = RND(ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],ss[1],78,0x5fcb6fab3ad6faec) ss[4], ss[0] = RND(ss[1],ss[2],ss[3],ss[4],ss[5],ss[6],ss[7],ss[0],79,0x6c44198c4a475817) dig = [] for i, x in enumerate(sha_info['digest']): dig.append( (x + ss[i]) & 0xffffffffffffffff ) sha_info['digest'] = dig def sha_init(): sha_info = new_shaobject() sha_info['digest'] = [ 0x6a09e667f3bcc908, 0xbb67ae8584caa73b, 0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1, 0x510e527fade682d1, 0x9b05688c2b3e6c1f, 0x1f83d9abfb41bd6b, 0x5be0cd19137e2179] sha_info['count_lo'] = 0 sha_info['count_hi'] = 0 sha_info['local'] = 0 sha_info['digestsize'] = 64 return sha_info def sha384_init(): sha_info = new_shaobject() sha_info['digest'] = [ 0xcbbb9d5dc1059ed8, 0x629a292a367cd507, 0x9159015a3070dd17, 0x152fecd8f70e5939, 0x67332667ffc00b31, 0x8eb44a8768581511, 0xdb0c2e0d64f98fa7, 0x47b5481dbefa4fa4] sha_info['count_lo'] = 0 sha_info['count_hi'] = 0 sha_info['local'] = 0 sha_info['digestsize'] = 48 return sha_info def getbuf(s): if isinstance(s, str): return s elif isinstance(s, unicode): return str(s) else: return buffer(s) def sha_update(sha_info, buffer): count = len(buffer) buffer_idx = 0 clo = (sha_info['count_lo'] + (count << 3)) & 0xffffffff if clo < sha_info['count_lo']: sha_info['count_hi'] += 1 sha_info['count_lo'] = clo sha_info['count_hi'] += (count >> 29) if sha_info['local']: i = SHA_BLOCKSIZE - sha_info['local'] if i > count: i = count # copy buffer for x in enumerate(buffer[buffer_idx:buffer_idx+i]): sha_info['data'][sha_info['local']+x[0]] = struct.unpack('B', x[1])[0] count -= i buffer_idx += i sha_info['local'] += i if sha_info['local'] == SHA_BLOCKSIZE: sha_transform(sha_info) sha_info['local'] = 0 else: return while count >= SHA_BLOCKSIZE: # copy buffer sha_info['data'] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + SHA_BLOCKSIZE]] count -= SHA_BLOCKSIZE buffer_idx += SHA_BLOCKSIZE sha_transform(sha_info) # copy buffer pos = sha_info['local'] sha_info['data'][pos:pos+count] = [struct.unpack('B',c)[0] for c in buffer[buffer_idx:buffer_idx + count]] sha_info['local'] = count def sha_final(sha_info): lo_bit_count = sha_info['count_lo'] hi_bit_count = sha_info['count_hi'] count = (lo_bit_count >> 3) & 0x7f sha_info['data'][count] = 0x80; count += 1 if count > SHA_BLOCKSIZE - 16: # zero the bytes in data after the count sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) sha_transform(sha_info) # zero bytes in data sha_info['data'] = [0] * SHA_BLOCKSIZE else: sha_info['data'] = sha_info['data'][:count] + ([0] * (SHA_BLOCKSIZE - count)) sha_info['data'][112] = 0; sha_info['data'][113] = 0; sha_info['data'][114] = 0; sha_info['data'][115] = 0; sha_info['data'][116] = 0; sha_info['data'][117] = 0; sha_info['data'][118] = 0; sha_info['data'][119] = 0; sha_info['data'][120] = (hi_bit_count >> 24) & 0xff sha_info['data'][121] = (hi_bit_count >> 16) & 0xff sha_info['data'][122] = (hi_bit_count >> 8) & 0xff sha_info['data'][123] = (hi_bit_count >> 0) & 0xff sha_info['data'][124] = (lo_bit_count >> 24) & 0xff sha_info['data'][125] = (lo_bit_count >> 16) & 0xff sha_info['data'][126] = (lo_bit_count >> 8) & 0xff sha_info['data'][127] = (lo_bit_count >> 0) & 0xff sha_transform(sha_info) dig = [] for i in sha_info['digest']: dig.extend([ ((i>>56) & 0xff), ((i>>48) & 0xff), ((i>>40) & 0xff), ((i>>32) & 0xff), ((i>>24) & 0xff), ((i>>16) & 0xff), ((i>>8) & 0xff), (i & 0xff) ]) return ''.join([chr(i) for i in dig]) class sha512(object): digest_size = digestsize = SHA_DIGESTSIZE block_size = SHA_BLOCKSIZE def __init__(self, s=None): self._sha = sha_init() if s: sha_update(self._sha, getbuf(s)) def update(self, s): sha_update(self._sha, getbuf(s)) def digest(self): return sha_final(self._sha.copy())[:self._sha['digestsize']] def hexdigest(self): return ''.join([('0%x' % ord(i))[-2:] for i in self.digest()]) def copy(self): new = sha512.__new__(sha512) new._sha = self._sha.copy() return new class sha384(sha512): digest_size = digestsize = 48 def __init__(self, s=None): self._sha = sha384_init() if s: sha_update(self._sha, getbuf(s)) def copy(self): new = sha384.__new__(sha384) new._sha = self._sha.copy() return new def test(): # import _sha512 a_str = "just a test string" assert sha512().hexdigest() == sha512().hexdigest() assert sha512(a_str).hexdigest() == sha512(a_str).hexdigest() assert sha512(a_str*7).hexdigest() == sha512(a_str*7).hexdigest() s = sha512(a_str) s.update(a_str) assert sha512(a_str+a_str).hexdigest() == s.hexdigest() if __name__ == "__main__": test() ================================================ FILE: third_party/pypy/_sre.py ================================================ # NOT_RPYTHON """ A pure Python reimplementation of the _sre module from CPython 2.4 Copyright 2005 Nik Haldimann, licensed under the MIT license This code is based on material licensed under CNRI's Python 1.6 license and copyrighted by: Copyright (c) 1997-2001 by Secret Labs AB """ #import array import sys import operator # # TODO: Support from foo import * syntax. import sre_constants for name in sre_constants.__all__: globals()[name] = getattr(sre_constants, name) # Identifying as _sre from Python 2.3 or 2.4 #if sys.version_info[:2] >= (2, 4): MAGIC = 20031017 #else: # MAGIC = 20030419 # In _sre.c this is bytesize of the code word type of the C implementation. # There it's 2 for normal Python builds and more for wide unicode builds (large # enough to hold a 32-bit UCS-4 encoded character). Since here in pure Python # we only see re bytecodes as Python longs, we shouldn't have to care about the # codesize. But sre_compile will compile some stuff differently depending on the # codesize (e.g., charsets). # starting with python 3.3 CODESIZE is 4 CODESIZE = 2 copyright = "_sre.py 2.4c Copyright 2005 by Nik Haldimann" def getcodesize(): return CODESIZE def compile(pattern, flags, code, groups=0, groupindex={}, indexgroup=[None]): """Compiles (or rather just converts) a pattern descriptor to a SRE_Pattern object. Actual compilation to opcodes happens in sre_compile.""" return SRE_Pattern(pattern, flags, code, groups, groupindex, indexgroup) def getlower(char_ord, flags): if (char_ord < 128) or (flags & SRE_FLAG_UNICODE) \ or (flags & SRE_FLAG_LOCALE and char_ord < 256): # return ord(unichr(char_ord).lower()) return ord(chr(char_ord).lower()) else: return char_ord class SRE_Pattern(object): def __init__(self, pattern, flags, code, groups=0, groupindex={}, indexgroup=[None]): self.pattern = pattern self.flags = flags self.groups = groups self.groupindex = groupindex # Maps group names to group indices self._indexgroup = indexgroup # Maps indices to group names self._code = code def match(self, string, pos=0, endpos=sys.maxint): """If zero or more characters at the beginning of string match this regular expression, return a corresponding MatchObject instance. Return None if the string does not match the pattern.""" state = _State(string, pos, endpos, self.flags) if state.match(self._code): return SRE_Match(self, state) else: return None def search(self, string, pos=0, endpos=sys.maxint): """Scan through string looking for a location where this regular expression produces a match, and return a corresponding MatchObject instance. Return None if no position in the string matches the pattern.""" state = _State(string, pos, endpos, self.flags) if state.search(self._code): return SRE_Match(self, state) else: return None def findall(self, string, pos=0, endpos=sys.maxint): """Return a list of all non-overlapping matches of pattern in string.""" matchlist = [] state = _State(string, pos, endpos, self.flags) while state.start <= state.end: state.reset() state.string_position = state.start if not state.search(self._code): break match = SRE_Match(self, state) if self.groups == 0 or self.groups == 1: item = match.group(self.groups) else: item = match.groups("") matchlist.append(item) if state.string_position == state.start: state.start += 1 else: state.start = state.string_position return matchlist def _subx(self, template, string, count=0, subn=False): filter = template if not callable(template) and "\\" in template: # handle non-literal strings ; hand it over to the template compiler raise NotImplementedError() state = _State(string, 0, sys.maxint, self.flags) sublist = [] n = last_pos = 0 while not count or n < count: state.reset() state.string_position = state.start if not state.search(self._code): break if last_pos < state.start: sublist.append(string[last_pos:state.start]) if not (last_pos == state.start and last_pos == state.string_position and n > 0): # the above ignores empty matches on latest position if callable(filter): sublist.append(filter(SRE_Match(self, state))) else: sublist.append(filter) last_pos = state.string_position n += 1 if state.string_position == state.start: state.start += 1 else: state.start = state.string_position if last_pos < state.end: sublist.append(string[last_pos:state.end]) item = "".join(sublist) if subn: return item, n else: return item def sub(self, repl, string, count=0): """Return the string obtained by replacing the leftmost non-overlapping occurrences of pattern in string by the replacement repl.""" return self._subx(repl, string, count, False) def subn(self, repl, string, count=0): """Return the tuple (new_string, number_of_subs_made) found by replacing the leftmost non-overlapping occurrences of pattern with the replacement repl.""" return self._subx(repl, string, count, True) def split(self, string, maxsplit=0): """Split string by the occurrences of pattern.""" splitlist = [] state = _State(string, 0, sys.maxint, self.flags) n = 0 last = state.start while not maxsplit or n < maxsplit: state.reset() state.string_position = state.start if not state.search(self._code): break if state.start == state.string_position: # zero-width match if last == state.end: # or end of string break state.start += 1 continue splitlist.append(string[last:state.start]) # add groups (if any) if self.groups: match = SRE_Match(self, state) # TODO: Use .extend once it is implemented. # splitlist.extend(list(match.groups(None))) splitlist += (list(match.groups(None))) n += 1 last = state.start = state.string_position splitlist.append(string[last:state.end]) return splitlist def finditer(self, string, pos=0, endpos=sys.maxint): """Return a list of all non-overlapping matches of pattern in string.""" scanner = self.scanner(string, pos, endpos) return iter(scanner.search, None) def scanner(self, string, start=0, end=sys.maxint): return SRE_Scanner(self, string, start, end) def __copy__(self): raise TypeError, "cannot copy this pattern object" def __deepcopy__(self): raise TypeError, "cannot copy this pattern object" class SRE_Scanner(object): """Undocumented scanner interface of sre.""" def __init__(self, pattern, string, start, end): self.pattern = pattern self._state = _State(string, start, end, self.pattern.flags) def _match_search(self, matcher): state = self._state state.reset() state.string_position = state.start match = None if matcher(self.pattern._code): match = SRE_Match(self.pattern, state) if match is None or state.string_position == state.start: state.start += 1 else: state.start = state.string_position return match def match(self): return self._match_search(self._state.match) def search(self): return self._match_search(self._state.search) class SRE_Match(object): def __init__(self, pattern, state): self.re = pattern self.string = state.string self.pos = state.pos self.endpos = state.end self.lastindex = state.lastindex if self.lastindex < 0: self.lastindex = None self.regs = self._create_regs(state) if pattern._indexgroup and 0 <= self.lastindex < len(pattern._indexgroup): # The above upper-bound check should not be necessary, as the re # compiler is supposed to always provide an _indexgroup list long # enough. But the re.Scanner class seems to screw up something # there, test_scanner in test_re won't work without upper-bound # checking. XXX investigate this and report bug to CPython. self.lastgroup = pattern._indexgroup[self.lastindex] else: self.lastgroup = None def _create_regs(self, state): """Creates a tuple of index pairs representing matched groups.""" regs = [(state.start, state.string_position)] for group in range(self.re.groups): mark_index = 2 * group if mark_index + 1 < len(state.marks) \ and state.marks[mark_index] is not None \ and state.marks[mark_index + 1] is not None: regs.append((state.marks[mark_index], state.marks[mark_index + 1])) else: regs.append((-1, -1)) return tuple(regs) def _get_index(self, group): if isinstance(group, int): if group >= 0 and group <= self.re.groups: return group else: if group in self.re.groupindex: return self.re.groupindex[group] raise IndexError("no such group") def _get_slice(self, group, default): group_indices = self.regs[group] if group_indices[0] >= 0: return self.string[group_indices[0]:group_indices[1]] else: return default def start(self, group=0): """Returns the indices of the start of the substring matched by group; group defaults to zero (meaning the whole matched substring). Returns -1 if group exists but did not contribute to the match.""" return self.regs[self._get_index(group)][0] def end(self, group=0): """Returns the indices of the end of the substring matched by group; group defaults to zero (meaning the whole matched substring). Returns -1 if group exists but did not contribute to the match.""" return self.regs[self._get_index(group)][1] def span(self, group=0): """Returns the 2-tuple (m.start(group), m.end(group)).""" return self.start(group), self.end(group) def expand(self, template): """Return the string obtained by doing backslash substitution and resolving group references on template.""" raise NotImplementedError def groups(self, default=None): """Returns a tuple containing all the subgroups of the match. The default argument is used for groups that did not participate in the match (defaults to None).""" groups = [] for indices in self.regs[1:]: if indices[0] >= 0: groups.append(self.string[indices[0]:indices[1]]) else: groups.append(default) return tuple(groups) def groupdict(self, default=None): """Return a dictionary containing all the named subgroups of the match. The default argument is used for groups that did not participate in the match (defaults to None).""" groupdict = {} for key, value in self.re.groupindex.items(): groupdict[key] = self._get_slice(value, default) return groupdict def group(self, *args): """Returns one or more subgroups of the match. Each argument is either a group index or a group name.""" if len(args) == 0: args = (0,) grouplist = [] for group in args: grouplist.append(self._get_slice(self._get_index(group), None)) if len(grouplist) == 1: return grouplist[0] else: return tuple(grouplist) def __copy__(): raise TypeError, "cannot copy this pattern object" def __deepcopy__(): raise TypeError, "cannot copy this pattern object" class _State(object): def __init__(self, string, start, end, flags): self.string = string if start < 0: start = 0 if end > len(string): end = len(string) self.start = start self.string_position = self.start self.end = end self.pos = start self.flags = flags self.reset() def reset(self): self.marks = [] self.lastindex = -1 self.marks_stack = [] self.context_stack = [] self.repeat = None def match(self, pattern_codes): # Optimization: Check string length. pattern_codes[3] contains the # minimum length for a string to possibly match. if pattern_codes[0] == OPCODES["info"] and pattern_codes[3]: if self.end - self.string_position < pattern_codes[3]: #_log("reject (got %d chars, need %d)" # % (self.end - self.string_position, pattern_codes[3])) return False dispatcher = _OpcodeDispatcher() self.context_stack.append(_MatchContext(self, pattern_codes)) has_matched = None while len(self.context_stack) > 0: context = self.context_stack[-1] has_matched = dispatcher.match(context) if has_matched is not None: # don't pop if context isn't done # TODO: use .pop once it is implemented # self.context_stack.pop() self.context_stack = self.context_stack[:-1] return has_matched def search(self, pattern_codes): flags = 0 if pattern_codes[0] == OPCODES["info"]: # optimization info block # <1=skip> <2=flags> <3=min> <4=max> <5=prefix info> if pattern_codes[2] & SRE_INFO_PREFIX and pattern_codes[5] > 1: return self.fast_search(pattern_codes) flags = pattern_codes[2] pattern_codes = pattern_codes[pattern_codes[1] + 1:] string_position = self.start if pattern_codes[0] == OPCODES["literal"]: # Special case: Pattern starts with a literal character. This is # used for short prefixes character = pattern_codes[1] while True: while string_position < self.end \ and ord(self.string[string_position]) != character: string_position += 1 if string_position >= self.end: return False self.start = string_position string_position += 1 self.string_position = string_position if flags & SRE_INFO_LITERAL: return True if self.match(pattern_codes[2:]): return True return False # General case while string_position <= self.end: self.reset() self.start = self.string_position = string_position if self.match(pattern_codes): return True string_position += 1 return False def fast_search(self, pattern_codes): """Skips forward in a string as fast as possible using information from an optimization info block.""" # pattern starts with a known prefix # <5=length> <6=skip> <7=prefix data> flags = pattern_codes[2] prefix_len = pattern_codes[5] prefix_skip = pattern_codes[6] # don't really know what this is good for prefix = pattern_codes[7:7 + prefix_len] overlap = pattern_codes[7 + prefix_len - 1:pattern_codes[1] + 1] pattern_codes = pattern_codes[pattern_codes[1] + 1:] i = 0 string_position = self.string_position while string_position < self.end: while True: if ord(self.string[string_position]) != prefix[i]: if i == 0: break else: i = overlap[i] else: i += 1 if i == prefix_len: # found a potential match self.start = string_position + 1 - prefix_len self.string_position = string_position + 1 \ - prefix_len + prefix_skip if flags & SRE_INFO_LITERAL: return True # matched all of pure literal pattern if self.match(pattern_codes[2 * prefix_skip:]): return True i = overlap[i] break string_position += 1 return False def set_mark(self, mark_nr, position): if mark_nr & 1: # This id marks the end of a group. self.lastindex = mark_nr / 2 + 1 if mark_nr >= len(self.marks): # TODO: Use .extend once it is implemented # self.marks.extend([None] * (mark_nr - len(self.marks) + 1)) self.marks += ([None] * (mark_nr - len(self.marks) + 1)) self.marks[mark_nr] = position def get_marks(self, group_index): marks_index = 2 * group_index if len(self.marks) > marks_index + 1: return self.marks[marks_index], self.marks[marks_index + 1] else: return None, None def marks_push(self): self.marks_stack.append((self.marks[:], self.lastindex)) def marks_pop(self): # TODO: Use .pop once implemented # self.marks, self.lastindex = self.marks_stack.pop() self.marks, self.lastindex = self.marks_stack[-1] self.marks_stack = self.marks_stack[:-1] def marks_pop_keep(self): self.marks, self.lastindex = self.marks_stack[-1] def marks_pop_discard(self): # TODO: Use .pop once implemented self.marks_stack = self.marks_stack[:-1] def lower(self, char_ord): return getlower(char_ord, self.flags) class _MatchContext(object): def __init__(self, state, pattern_codes): self.state = state self.pattern_codes = pattern_codes self.string_position = state.string_position self.code_position = 0 self.has_matched = None def push_new_context(self, pattern_offset): """Creates a new child context of this context and pushes it on the stack. pattern_offset is the offset off the current code position to start interpreting from.""" child_context = _MatchContext(self.state, self.pattern_codes[self.code_position + pattern_offset:]) self.state.context_stack.append(child_context) return child_context def peek_char(self, peek=0): return self.state.string[self.string_position + peek] def skip_char(self, skip_count): self.string_position += skip_count def remaining_chars(self): return self.state.end - self.string_position def peek_code(self, peek=0): return self.pattern_codes[self.code_position + peek] def skip_code(self, skip_count): self.code_position += skip_count def remaining_codes(self): return len(self.pattern_codes) - self.code_position def at_beginning(self): return self.string_position == 0 def at_end(self): return self.string_position == self.state.end def at_linebreak(self): return not self.at_end() and _is_linebreak(self.peek_char()) def at_boundary(self, word_checker): if self.at_beginning() and self.at_end(): return False that = not self.at_beginning() and word_checker(self.peek_char(-1)) this = not self.at_end() and word_checker(self.peek_char()) return this != that class _RepeatContext(_MatchContext): def __init__(self, context): _MatchContext.__init__(self, context.state, context.pattern_codes[context.code_position:]) self.count = -1 self.previous = context.state.repeat self.last_position = None class _Dispatcher(object): DISPATCH_TABLE = None def dispatch(self, code, context): method = self.DISPATCH_TABLE.get(code, self.__class__.unknown) return method(self, context) def unknown(self, code, ctx): raise NotImplementedError() def build_dispatch_table(cls, code_dict, method_prefix): if cls.DISPATCH_TABLE is not None: return table = {} for key, value in code_dict.items(): if hasattr(cls, "%s%s" % (method_prefix, key)): table[value] = getattr(cls, "%s%s" % (method_prefix, key)) cls.DISPATCH_TABLE = table build_dispatch_table = classmethod(build_dispatch_table) class _OpcodeDispatcher(_Dispatcher): def __init__(self): self.executing_contexts = {} self.at_dispatcher = _AtcodeDispatcher() self.ch_dispatcher = _ChcodeDispatcher() self.set_dispatcher = _CharsetDispatcher() def match(self, context): """Returns True if the current context matches, False if it doesn't and None if matching is not finished, ie must be resumed after child contexts have been matched.""" while context.remaining_codes() > 0 and context.has_matched is None: opcode = context.peek_code() if not self.dispatch(opcode, context): return None if context.has_matched is None: context.has_matched = False return context.has_matched def dispatch(self, opcode, context): """Dispatches a context on a given opcode. Returns True if the context is done matching, False if it must be resumed when next encountered.""" if id(context) in self.executing_contexts: generator = self.executing_contexts[id(context)] del self.executing_contexts[id(context)] has_finished = generator.next() else: method = self.DISPATCH_TABLE.get(opcode, _OpcodeDispatcher.unknown) has_finished = method(self, context) if hasattr(has_finished, "next"): # avoid using the types module generator = has_finished has_finished = generator.next() if not has_finished: self.executing_contexts[id(context)] = generator return has_finished def op_success(self, ctx): # end of pattern #self._log(ctx, "SUCCESS") ctx.state.string_position = ctx.string_position ctx.has_matched = True return True def op_failure(self, ctx): # immediate failure #self._log(ctx, "FAILURE") ctx.has_matched = False return True def general_op_literal(self, ctx, compare, decorate=lambda x: x): if ctx.at_end() or not compare(decorate(ord(ctx.peek_char())), decorate(ctx.peek_code(1))): ctx.has_matched = False ctx.skip_code(2) ctx.skip_char(1) def op_literal(self, ctx): # match literal string # #self._log(ctx, "LITERAL", ctx.peek_code(1)) self.general_op_literal(ctx, operator.eq) return True def op_not_literal(self, ctx): # match anything that is not the given literal character # #self._log(ctx, "NOT_LITERAL", ctx.peek_code(1)) self.general_op_literal(ctx, operator.ne) return True def op_literal_ignore(self, ctx): # match literal regardless of case # #self._log(ctx, "LITERAL_IGNORE", ctx.peek_code(1)) self.general_op_literal(ctx, operator.eq, ctx.state.lower) return True def op_not_literal_ignore(self, ctx): # match literal regardless of case # #self._log(ctx, "LITERAL_IGNORE", ctx.peek_code(1)) self.general_op_literal(ctx, operator.ne, ctx.state.lower) return True def op_at(self, ctx): # match at given position # #self._log(ctx, "AT", ctx.peek_code(1)) if not self.at_dispatcher.dispatch(ctx.peek_code(1), ctx): ctx.has_matched = False return True ctx.skip_code(2) return True def op_category(self, ctx): # match at given category # #self._log(ctx, "CATEGORY", ctx.peek_code(1)) if ctx.at_end() or not self.ch_dispatcher.dispatch(ctx.peek_code(1), ctx): ctx.has_matched = False return True ctx.skip_code(2) ctx.skip_char(1) return True def op_any(self, ctx): # match anything (except a newline) # #self._log(ctx, "ANY") if ctx.at_end() or ctx.at_linebreak(): ctx.has_matched = False return True ctx.skip_code(1) ctx.skip_char(1) return True def op_any_all(self, ctx): # match anything # #self._log(ctx, "ANY_ALL") if ctx.at_end(): ctx.has_matched = False return True ctx.skip_code(1) ctx.skip_char(1) return True def general_op_in(self, ctx, decorate=lambda x: x): #self._log(ctx, "OP_IN") if ctx.at_end(): ctx.has_matched = False return skip = ctx.peek_code(1) ctx.skip_code(2) # set op pointer to the set code if not self.check_charset(ctx, decorate(ord(ctx.peek_char()))): ctx.has_matched = False return ctx.skip_code(skip - 1) ctx.skip_char(1) def op_in(self, ctx): # match set member (or non_member) # #self._log(ctx, "OP_IN") self.general_op_in(ctx) return True def op_in_ignore(self, ctx): # match set member (or non_member), disregarding case of current char # #self._log(ctx, "OP_IN_IGNORE") self.general_op_in(ctx, ctx.state.lower) return True def op_jump(self, ctx): # jump forward # #self._log(ctx, "JUMP", ctx.peek_code(1)) ctx.skip_code(ctx.peek_code(1) + 1) return True # skip info # op_info = op_jump def op_mark(self, ctx): # set mark # #self._log(ctx, "OP_MARK", ctx.peek_code(1)) ctx.state.set_mark(ctx.peek_code(1), ctx.string_position) ctx.skip_code(2) return True def op_branch(self, ctx): # alternation # <0=skip> code ... #self._log(ctx, "BRANCH") ctx.state.marks_push() ctx.skip_code(1) current_branch_length = ctx.peek_code(0) while current_branch_length: # The following tries to shortcut branches starting with a # (unmatched) literal. _sre.c also shortcuts charsets here. if not (ctx.peek_code(1) == OPCODES["literal"] and \ (ctx.at_end() or ctx.peek_code(2) != ord(ctx.peek_char()))): ctx.state.string_position = ctx.string_position child_context = ctx.push_new_context(1) yield False if child_context.has_matched: ctx.has_matched = True yield True ctx.state.marks_pop_keep() ctx.skip_code(current_branch_length) current_branch_length = ctx.peek_code(0) ctx.state.marks_pop_discard() ctx.has_matched = False yield True def op_repeat_one(self, ctx): # match repeated sequence (maximizing). # this operator only works if the repeated item is exactly one character # wide, and we're not already collecting backtracking points. # <1=min> <2=max> item tail mincount = ctx.peek_code(2) maxcount = ctx.peek_code(3) #self._log(ctx, "REPEAT_ONE", mincount, maxcount) if ctx.remaining_chars() < mincount: ctx.has_matched = False yield True ctx.state.string_position = ctx.string_position count = self.count_repetitions(ctx, maxcount) ctx.skip_char(count) if count < mincount: ctx.has_matched = False yield True if ctx.peek_code(ctx.peek_code(1) + 1) == OPCODES["success"]: # tail is empty. we're finished ctx.state.string_position = ctx.string_position ctx.has_matched = True yield True ctx.state.marks_push() if ctx.peek_code(ctx.peek_code(1) + 1) == OPCODES["literal"]: # Special case: Tail starts with a literal. Skip positions where # the rest of the pattern cannot possibly match. char = ctx.peek_code(ctx.peek_code(1) + 2) while True: while count >= mincount and \ (ctx.at_end() or ord(ctx.peek_char()) != char): ctx.skip_char(-1) count -= 1 if count < mincount: break ctx.state.string_position = ctx.string_position child_context = ctx.push_new_context(ctx.peek_code(1) + 1) yield False if child_context.has_matched: ctx.has_matched = True yield True ctx.skip_char(-1) count -= 1 ctx.state.marks_pop_keep() else: # General case: backtracking while count >= mincount: ctx.state.string_position = ctx.string_position child_context = ctx.push_new_context(ctx.peek_code(1) + 1) yield False if child_context.has_matched: ctx.has_matched = True yield True ctx.skip_char(-1) count -= 1 ctx.state.marks_pop_keep() ctx.state.marks_pop_discard() ctx.has_matched = False yield True def op_min_repeat_one(self, ctx): # match repeated sequence (minimizing) # <1=min> <2=max> item tail mincount = ctx.peek_code(2) maxcount = ctx.peek_code(3) #self._log(ctx, "MIN_REPEAT_ONE", mincount, maxcount) if ctx.remaining_chars() < mincount: ctx.has_matched = False yield True ctx.state.string_position = ctx.string_position if mincount == 0: count = 0 else: count = self.count_repetitions(ctx, mincount) if count < mincount: ctx.has_matched = False yield True ctx.skip_char(count) if ctx.peek_code(ctx.peek_code(1) + 1) == OPCODES["success"]: # tail is empty. we're finished ctx.state.string_position = ctx.string_position ctx.has_matched = True yield True ctx.state.marks_push() while maxcount == MAXREPEAT or count <= maxcount: ctx.state.string_position = ctx.string_position child_context = ctx.push_new_context(ctx.peek_code(1) + 1) yield False if child_context.has_matched: ctx.has_matched = True yield True ctx.state.string_position = ctx.string_position if self.count_repetitions(ctx, 1) == 0: break ctx.skip_char(1) count += 1 ctx.state.marks_pop_keep() ctx.state.marks_pop_discard() ctx.has_matched = False yield True def op_repeat(self, ctx): # create repeat context. all the hard work is done by the UNTIL # operator (MAX_UNTIL, MIN_UNTIL) # <1=min> <2=max> item tail #self._log(ctx, "REPEAT", ctx.peek_code(2), ctx.peek_code(3)) repeat = _RepeatContext(ctx) ctx.state.repeat = repeat ctx.state.string_position = ctx.string_position child_context = ctx.push_new_context(ctx.peek_code(1) + 1) yield False ctx.state.repeat = repeat.previous ctx.has_matched = child_context.has_matched yield True def op_max_until(self, ctx): # maximizing repeat # <1=min> <2=max> item tail repeat = ctx.state.repeat if repeat is None: raise RuntimeError("Internal re error: MAX_UNTIL without REPEAT.") mincount = repeat.peek_code(2) maxcount = repeat.peek_code(3) ctx.state.string_position = ctx.string_position count = repeat.count + 1 #self._log(ctx, "MAX_UNTIL", count) if count < mincount: # not enough matches repeat.count = count child_context = repeat.push_new_context(4) yield False ctx.has_matched = child_context.has_matched if not ctx.has_matched: repeat.count = count - 1 ctx.state.string_position = ctx.string_position yield True if (count < maxcount or maxcount == MAXREPEAT) \ and ctx.state.string_position != repeat.last_position: # we may have enough matches, if we can match another item, do so repeat.count = count ctx.state.marks_push() save_last_position = repeat.last_position # zero-width match protection repeat.last_position = ctx.state.string_position child_context = repeat.push_new_context(4) yield False repeat.last_position = save_last_position if child_context.has_matched: ctx.state.marks_pop_discard() ctx.has_matched = True yield True ctx.state.marks_pop() repeat.count = count - 1 ctx.state.string_position = ctx.string_position # cannot match more repeated items here. make sure the tail matches ctx.state.repeat = repeat.previous child_context = ctx.push_new_context(1) yield False ctx.has_matched = child_context.has_matched if not ctx.has_matched: ctx.state.repeat = repeat ctx.state.string_position = ctx.string_position yield True def op_min_until(self, ctx): # minimizing repeat # <1=min> <2=max> item tail repeat = ctx.state.repeat if repeat is None: raise RuntimeError("Internal re error: MIN_UNTIL without REPEAT.") mincount = repeat.peek_code(2) maxcount = repeat.peek_code(3) ctx.state.string_position = ctx.string_position count = repeat.count + 1 #self._log(ctx, "MIN_UNTIL", count) if count < mincount: # not enough matches repeat.count = count child_context = repeat.push_new_context(4) yield False ctx.has_matched = child_context.has_matched if not ctx.has_matched: repeat.count = count - 1 ctx.state.string_position = ctx.string_position yield True # see if the tail matches ctx.state.marks_push() ctx.state.repeat = repeat.previous child_context = ctx.push_new_context(1) yield False if child_context.has_matched: ctx.has_matched = True yield True ctx.state.repeat = repeat ctx.state.string_position = ctx.string_position ctx.state.marks_pop() # match more until tail matches if count >= maxcount and maxcount != MAXREPEAT: ctx.has_matched = False yield True repeat.count = count child_context = repeat.push_new_context(4) yield False ctx.has_matched = child_context.has_matched if not ctx.has_matched: repeat.count = count - 1 ctx.state.string_position = ctx.string_position yield True def general_op_groupref(self, ctx, decorate=lambda x: x): group_start, group_end = ctx.state.get_marks(ctx.peek_code(1)) if group_start is None or group_end is None or group_end < group_start: ctx.has_matched = False return True while group_start < group_end: if ctx.at_end() or decorate(ord(ctx.peek_char())) \ != decorate(ord(ctx.state.string[group_start])): ctx.has_matched = False return True group_start += 1 ctx.skip_char(1) ctx.skip_code(2) return True def op_groupref(self, ctx): # match backreference # #self._log(ctx, "GROUPREF", ctx.peek_code(1)) return self.general_op_groupref(ctx) def op_groupref_ignore(self, ctx): # match backreference case-insensitive # #self._log(ctx, "GROUPREF_IGNORE", ctx.peek_code(1)) return self.general_op_groupref(ctx, ctx.state.lower) def op_groupref_exists(self, ctx): # codeyes codeno ... #self._log(ctx, "GROUPREF_EXISTS", ctx.peek_code(1)) group_start, group_end = ctx.state.get_marks(ctx.peek_code(1)) if group_start is None or group_end is None or group_end < group_start: ctx.skip_code(ctx.peek_code(2) + 1) else: ctx.skip_code(3) return True def op_assert(self, ctx): # assert subpattern # #self._log(ctx, "ASSERT", ctx.peek_code(2)) ctx.state.string_position = ctx.string_position - ctx.peek_code(2) if ctx.state.string_position < 0: ctx.has_matched = False yield True child_context = ctx.push_new_context(3) yield False if child_context.has_matched: ctx.skip_code(ctx.peek_code(1) + 1) else: ctx.has_matched = False yield True def op_assert_not(self, ctx): # assert not subpattern # #self._log(ctx, "ASSERT_NOT", ctx.peek_code(2)) ctx.state.string_position = ctx.string_position - ctx.peek_code(2) if ctx.state.string_position >= 0: child_context = ctx.push_new_context(3) yield False if child_context.has_matched: ctx.has_matched = False yield True ctx.skip_code(ctx.peek_code(1) + 1) yield True def unknown(self, ctx): #self._log(ctx, "UNKNOWN", ctx.peek_code()) raise RuntimeError("Internal re error. Unknown opcode: %s" % ctx.peek_code()) def check_charset(self, ctx, char): """Checks whether a character matches set of arbitrary length. Assumes the code pointer is at the first member of the set.""" self.set_dispatcher.reset(char) save_position = ctx.code_position result = None while result is None: result = self.set_dispatcher.dispatch(ctx.peek_code(), ctx) ctx.code_position = save_position return result def count_repetitions(self, ctx, maxcount): """Returns the number of repetitions of a single item, starting from the current string position. The code pointer is expected to point to a REPEAT_ONE operation (with the repeated 4 ahead).""" count = 0 real_maxcount = ctx.state.end - ctx.string_position if maxcount < real_maxcount and maxcount != MAXREPEAT: real_maxcount = maxcount # XXX could special case every single character pattern here, as in C. # This is a general solution, a bit hackisch, but works and should be # efficient. code_position = ctx.code_position string_position = ctx.string_position ctx.skip_code(4) reset_position = ctx.code_position while count < real_maxcount: # this works because the single character pattern is followed by # a success opcode ctx.code_position = reset_position self.dispatch(ctx.peek_code(), ctx) if ctx.has_matched is False: # could be None as well break count += 1 ctx.has_matched = None ctx.code_position = code_position ctx.string_position = string_position return count def _log(self, context, opname, *args): arg_string = ("%s " * len(args)) % args _log("|%s|%s|%s %s" % (context.pattern_codes, context.string_position, opname, arg_string)) _OpcodeDispatcher.build_dispatch_table(OPCODES, "op_") class _CharsetDispatcher(_Dispatcher): def __init__(self): self.ch_dispatcher = _ChcodeDispatcher() def reset(self, char): self.char = char self.ok = True def set_failure(self, ctx): return not self.ok def set_literal(self, ctx): # if ctx.peek_code(1) == self.char: return self.ok else: ctx.skip_code(2) def set_category(self, ctx): # if self.ch_dispatcher.dispatch(ctx.peek_code(1), ctx): return self.ok else: ctx.skip_code(2) def set_charset(self, ctx): # (16 bits per code word) char_code = self.char ctx.skip_code(1) # point to beginning of bitmap if CODESIZE == 2: if char_code < 256 and ctx.peek_code(char_code >> 4) \ & (1 << (char_code & 15)): return self.ok ctx.skip_code(16) # skip bitmap else: if char_code < 256 and ctx.peek_code(char_code >> 5) \ & (1 << (char_code & 31)): return self.ok ctx.skip_code(8) # skip bitmap def set_range(self, ctx): # if ctx.peek_code(1) <= self.char <= ctx.peek_code(2): return self.ok ctx.skip_code(3) def set_negate(self, ctx): self.ok = not self.ok ctx.skip_code(1) def set_bigcharset(self, ctx): # <256 blockindices> char_code = self.char count = ctx.peek_code(1) ctx.skip_code(2) if char_code < 65536: block_index = char_code >> 8 # NB: there are CODESIZE block indices per bytecode # a = array.array("B") a = [] # a.fromstring(array.array(CODESIZE == 2 and "H" or "I", # [ctx.peek_code(block_index / CODESIZE)]).tostring()) a += [ctx.peek_code(block_index // CODESIZE)] block = a[block_index % CODESIZE] ctx.skip_code(256 / CODESIZE) # skip block indices block_value = ctx.peek_code(block * (32 / CODESIZE) + ((char_code & 255) >> (CODESIZE == 2 and 4 or 5))) if block_value & (1 << (char_code & ((8 * CODESIZE) - 1))): return self.ok else: ctx.skip_code(256 / CODESIZE) # skip block indices ctx.skip_code(count * (32 / CODESIZE)) # skip blocks def unknown(self, ctx): return False _CharsetDispatcher.build_dispatch_table(OPCODES, "set_") class _AtcodeDispatcher(_Dispatcher): def at_beginning(self, ctx): return ctx.at_beginning() at_beginning_string = at_beginning def at_beginning_line(self, ctx): return ctx.at_beginning() or _is_linebreak(ctx.peek_char(-1)) def at_end(self, ctx): return (ctx.remaining_chars() == 1 and ctx.at_linebreak()) or ctx.at_end() def at_end_line(self, ctx): return ctx.at_linebreak() or ctx.at_end() def at_end_string(self, ctx): return ctx.at_end() def at_boundary(self, ctx): return ctx.at_boundary(_is_word) def at_non_boundary(self, ctx): return not ctx.at_boundary(_is_word) def at_loc_boundary(self, ctx): return ctx.at_boundary(_is_loc_word) def at_loc_non_boundary(self, ctx): return not ctx.at_boundary(_is_loc_word) def at_uni_boundary(self, ctx): return ctx.at_boundary(_is_uni_word) def at_uni_non_boundary(self, ctx): return not ctx.at_boundary(_is_uni_word) def unknown(self, ctx): return False _AtcodeDispatcher.build_dispatch_table(ATCODES, "") class _ChcodeDispatcher(_Dispatcher): def category_digit(self, ctx): return _is_digit(ctx.peek_char()) def category_not_digit(self, ctx): return not _is_digit(ctx.peek_char()) def category_space(self, ctx): return _is_space(ctx.peek_char()) def category_not_space(self, ctx): return not _is_space(ctx.peek_char()) def category_word(self, ctx): return _is_word(ctx.peek_char()) def category_not_word(self, ctx): return not _is_word(ctx.peek_char()) def category_linebreak(self, ctx): return _is_linebreak(ctx.peek_char()) def category_not_linebreak(self, ctx): return not _is_linebreak(ctx.peek_char()) def category_loc_word(self, ctx): return _is_loc_word(ctx.peek_char()) def category_loc_not_word(self, ctx): return not _is_loc_word(ctx.peek_char()) def category_uni_digit(self, ctx): return ctx.peek_char().isdigit() def category_uni_not_digit(self, ctx): return not ctx.peek_char().isdigit() def category_uni_space(self, ctx): return ctx.peek_char().isspace() def category_uni_not_space(self, ctx): return not ctx.peek_char().isspace() def category_uni_word(self, ctx): return _is_uni_word(ctx.peek_char()) def category_uni_not_word(self, ctx): return not _is_uni_word(ctx.peek_char()) def category_uni_linebreak(self, ctx): return ord(ctx.peek_char()) in _uni_linebreaks def category_uni_not_linebreak(self, ctx): return ord(ctx.peek_char()) not in _uni_linebreaks def unknown(self, ctx): return False _ChcodeDispatcher.build_dispatch_table(CHCODES, "") _ascii_char_info = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 16, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0 ] def _is_digit(char): code = ord(char) return code < 128 and _ascii_char_info[code] & 1 def _is_space(char): code = ord(char) return code < 128 and _ascii_char_info[code] & 2 def _is_word(char): # NB: non-ASCII chars aren't words according to _sre.c code = ord(char) return code < 128 and _ascii_char_info[code] & 16 def _is_loc_word(char): return (not (ord(char) & ~255) and char.isalnum()) or char == '_' def _is_uni_word(char): return unichr(ord(char)).isalnum() or char == '_' def _is_linebreak(char): return char == "\n" # Static list of all unicode codepoints reported by Py_UNICODE_ISLINEBREAK. _uni_linebreaks = [10, 13, 28, 29, 30, 133, 8232, 8233] def _log(message): if 0: print message ================================================ FILE: third_party/pypy/_struct.py ================================================ # # This module is a pure Python version of pypy.module.struct. # It is only imported if the vastly faster pypy.module.struct is not # compiled in. For now we keep this version for reference and # because pypy.module.struct is not ootype-backend-friendly yet. # """Functions to convert between Python values and C structs. Python strings are used to hold the data representing the C struct and also as format strings to describe the layout of data in the C struct. The optional first format char indicates byte order, size and alignment: @: native order, size & alignment (default) =: native order, std. size & alignment <: little-endian, std. size & alignment >: big-endian, std. size & alignment !: same as > The remaining chars indicate types of args and must match exactly; these can be preceded by a decimal repeat count: x: pad byte (no data); c:char; b:signed byte; B:unsigned byte; h:short; H:unsigned short; i:int; I:unsigned int; l:long; L:unsigned long; f:float; d:double. Special cases (preceding decimal count indicates length): s:string (array of char); p: pascal string (with count byte). Special case (only available in native format): P:an integer type that is wide enough to hold a pointer. Special case (not in native mode unless 'long long' in platform C): q:long long; Q:unsigned long long Whitespace between formats is ignored. The variable struct.error is an exception raised on errors.""" import math import sys # TODO: XXX Find a way to get information on native sizes and alignments class StructError(Exception): pass error = StructError bytes = str def unpack_int(data, index, size, le): _bytes = [b for b in data[index:index + size]] if le == 'little': _bytes.reverse() number = 0 for b in _bytes: number = number << 8 | b return int(number) def unpack_signed_int(data, index, size, le): number = unpack_int(data, index, size, le) max = (1 << (size * 8)) if number > (1 << (size * 8 - 1)) - 1: number = int(-1 * (max - number)) return number INFINITY = 1e200 * 1e200 NAN = INFINITY / INFINITY def unpack_char(data, index, size, le): return data[index:index + size] def pack_int(number, size, le): x = number res = [] for i in range(size): res.append(x & 0xff) x = x >> 8 if le == 'big': res.reverse() return ''.join(chr(x) for x in res) def pack_signed_int(number, size, le): if not isinstance(number, int): raise StructError("argument for i,I,l,L,q,Q,h,H must be integer") if number > (1 << (8 * size - 1)) - 1 or number < -1 * (1 << (8 * size - 1)): raise OverflowError("Number:%i too large to convert" % number) return pack_int(number, size, le) def pack_unsigned_int(number, size, le): if not isinstance(number, int): raise StructError("argument for i,I,l,L,q,Q,h,H must be integer") if number < 0: raise TypeError("can't convert negative long to unsigned") if number > (1 << (8 * size)) - 1: raise OverflowError("Number:%i too large to convert" % number) return pack_int(number, size, le) def pack_char(char, size, le): return str(char) def isinf(x): return x != 0.0 and x / 2 == x def isnan(v): return v != v * 1.0 or (v == 1.0 and v == 2.0) def pack_float(x, size, le): unsigned = float_pack(x, size) result = [] for i in range(8): result.append((unsigned >> (i * 8)) & 0xFF) if le == "big": result.reverse() return ''.join(chr(x) for x in result) def unpack_float(data, index, size, le): binary = [data[i] for i in range(index, index + 8)] if le == "big": binary.reverse() unsigned = 0 for i in range(8): # unsigned |= binary[i] << (i * 8) unsigned |= ord(binary[i]) << (i * 8) return float_unpack(unsigned, size, le) def round_to_nearest(x): """Python 3 style round: round a float x to the nearest int, but unlike the builtin Python 2.x round function: - return an int, not a float - do round-half-to-even, not round-half-away-from-zero. We assume that x is finite and nonnegative; except wrong results if you use this for negative x. """ int_part = int(x) frac_part = x - int_part if frac_part > 0.5 or frac_part == 0.5 and int_part & 1 == 1: int_part += 1 return int_part def float_unpack(Q, size, le): """Convert a 32-bit or 64-bit integer created by float_pack into a Python float.""" if size == 8: MIN_EXP = -1021 # = sys.float_info.min_exp MAX_EXP = 1024 # = sys.float_info.max_exp MANT_DIG = 53 # = sys.float_info.mant_dig BITS = 64 elif size == 4: MIN_EXP = -125 # C's FLT_MIN_EXP MAX_EXP = 128 # FLT_MAX_EXP MANT_DIG = 24 # FLT_MANT_DIG BITS = 32 else: raise ValueError("invalid size value") if Q >> BITS: raise ValueError("input out of range") # extract pieces sign = Q >> BITS - 1 exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1 mant = Q & ((1 << MANT_DIG - 1) - 1) if exp == MAX_EXP - MIN_EXP + 2: # nan or infinity result = float('nan') if mant else float('inf') elif exp == 0: # subnormal or zero result = math.ldexp(float(mant), MIN_EXP - MANT_DIG) else: # normal mant += 1 << MANT_DIG - 1 result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1) return -result if sign else result def float_pack(x, size): """Convert a Python float x into a 64-bit unsigned integer with the same byte representation.""" if size == 8: MIN_EXP = -1021 # = sys.float_info.min_exp MAX_EXP = 1024 # = sys.float_info.max_exp MANT_DIG = 53 # = sys.float_info.mant_dig BITS = 64 elif size == 4: MIN_EXP = -125 # C's FLT_MIN_EXP MAX_EXP = 128 # FLT_MAX_EXP MANT_DIG = 24 # FLT_MANT_DIG BITS = 32 else: raise ValueError("invalid size value") sign = math.copysign(1.0, x) < 0.0 if math.isinf(x): mant = 0 exp = MAX_EXP - MIN_EXP + 2 elif math.isnan(x): mant = 1 << (MANT_DIG - 2) # other values possible exp = MAX_EXP - MIN_EXP + 2 elif x == 0.0: mant = 0 exp = 0 else: m, e = math.frexp(abs(x)) # abs(x) == m * 2**e exp = e - (MIN_EXP - 1) if exp > 0: # Normal case. mant = round_to_nearest(m * (1 << MANT_DIG)) mant -= 1 << MANT_DIG - 1 else: # Subnormal case. if exp + MANT_DIG - 1 >= 0: mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1)) else: mant = 0 exp = 0 # Special case: rounding produced a MANT_DIG-bit mantissa. assert 0 <= mant <= 1 << MANT_DIG - 1 if mant == 1 << MANT_DIG - 1: mant = 0 exp += 1 # Raise on overflow (in some circumstances, may want to return # infinity instead). if exp >= MAX_EXP - MIN_EXP + 2: raise OverflowError("float too large to pack in this format") # check constraints assert 0 <= mant < 1 << MANT_DIG - 1 assert 0 <= exp <= MAX_EXP - MIN_EXP + 2 assert 0 <= sign <= 1 return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant big_endian_format = { 'x': {'size': 1, 'alignment': 0, 'pack': None, 'unpack': None}, 'b': {'size': 1, 'alignment': 0, 'pack': pack_signed_int, 'unpack': unpack_signed_int}, 'B': {'size': 1, 'alignment': 0, 'pack': pack_unsigned_int, 'unpack': unpack_int}, 'c': {'size': 1, 'alignment': 0, 'pack': pack_char, 'unpack': unpack_char}, 's': {'size': 1, 'alignment': 0, 'pack': None, 'unpack': None}, 'p': {'size': 1, 'alignment': 0, 'pack': None, 'unpack': None}, 'h': {'size': 2, 'alignment': 0, 'pack': pack_signed_int, 'unpack': unpack_signed_int}, 'H': {'size': 2, 'alignment': 0, 'pack': pack_unsigned_int, 'unpack': unpack_int}, 'i': {'size': 4, 'alignment': 0, 'pack': pack_signed_int, 'unpack': unpack_signed_int}, 'I': {'size': 4, 'alignment': 0, 'pack': pack_unsigned_int, 'unpack': unpack_int}, 'l': {'size': 4, 'alignment': 0, 'pack': pack_signed_int, 'unpack': unpack_signed_int}, 'L': {'size': 4, 'alignment': 0, 'pack': pack_unsigned_int, 'unpack': unpack_int}, 'q': {'size': 8, 'alignment': 0, 'pack': pack_signed_int, 'unpack': unpack_signed_int}, 'Q': {'size': 8, 'alignment': 0, 'pack': pack_unsigned_int, 'unpack': unpack_int}, 'f': {'size': 4, 'alignment': 0, 'pack': pack_float, 'unpack': unpack_float}, 'd': {'size': 8, 'alignment': 0, 'pack': pack_float, 'unpack': unpack_float}, } default = big_endian_format formatmode = {'<': (default, 'little'), '>': (default, 'big'), '!': (default, 'big'), '=': (default, sys.byteorder), '@': (default, sys.byteorder) } def getmode(fmt): try: formatdef, endianness = formatmode[fmt[0]] index = 1 except (IndexError, KeyError): formatdef, endianness = formatmode['@'] index = 0 return formatdef, endianness, index def getNum(fmt, i): num = None cur = fmt[i] while ('0' <= cur) and (cur <= '9'): if num == None: num = int(cur) else: num = 10 * num + int(cur) i += 1 cur = fmt[i] return num, i def calcsize(fmt): """calcsize(fmt) -> int Return size of C struct described by format string fmt. See struct.__doc__ for more on format strings.""" formatdef, endianness, i = getmode(fmt) num = 0 result = 0 while i < len(fmt): num, i = getNum(fmt, i) cur = fmt[i] try: format = formatdef[cur] except KeyError: raise StructError("%s is not a valid format" % cur) if num != None: result += num * format['size'] else: result += format['size'] num = 0 i += 1 return result def pack(fmt, *args): """pack(fmt, v1, v2, ...) -> string Return string containing values v1, v2, ... packed according to fmt. See struct.__doc__ for more on format strings.""" formatdef, endianness, i = getmode(fmt) args = list(args) n_args = len(args) result = [] while i < len(fmt): num, i = getNum(fmt, i) cur = fmt[i] try: format = formatdef[cur] except KeyError: raise StructError("%s is not a valid format" % cur) if num == None: num_s = 0 num = 1 else: num_s = num if cur == 'x': result += [b'\0' * num] elif cur == 's': if isinstance(args[0], bytes): padding = num - len(args[0]) result += [args[0][:num] + b'\0' * padding] args.pop(0) else: raise StructError("arg for string format not a string") elif cur == 'p': if isinstance(args[0], bytes): padding = num - len(args[0]) - 1 if padding > 0: result += [bytes([len(args[0])]) + args[0] [:num - 1] + b'\0' * padding] else: if num < 255: result += [bytes([num - 1]) + args[0][:num - 1]] else: result += [bytes([255]) + args[0][:num - 1]] args.pop(0) else: raise StructError("arg for string format not a string") else: if len(args) < num: raise StructError("insufficient arguments to pack") for var in args[:num]: result += [format['pack'](var, format['size'], endianness)] args = args[num:] num = None i += 1 if len(args) != 0: raise StructError("too many arguments for pack format") return b''.join(result) def unpack(fmt, data): """unpack(fmt, string) -> (v1, v2, ...) Unpack the string, containing packed C structure data, according to fmt. Requires len(string)==calcsize(fmt). See struct.__doc__ for more on format strings.""" formatdef, endianness, i = getmode(fmt) j = 0 num = 0 result = [] length = calcsize(fmt) if length != len(data): raise StructError("unpack str size does not match format") while i < len(fmt): num, i = getNum(fmt, i) cur = fmt[i] i += 1 try: format = formatdef[cur] except KeyError: raise StructError("%s is not a valid format" % cur) if not num: num = 1 if cur == 'x': j += num elif cur == 's': result.append(data[j:j + num]) j += num elif cur == 'p': n = data[j] if n >= num: n = num - 1 result.append(data[j + 1:j + n + 1]) j += num else: for n in range(num): result += [format['unpack'](data, j, format['size'], endianness)] j += format['size'] return tuple(result) def pack_into(fmt, buf, offset, *args): data = pack(fmt, *args) buffer(buf)[offset:offset + len(data)] = data def unpack_from(fmt, buf, offset=0): size = calcsize(fmt) data = buffer(buf)[offset:offset + size] if len(data) != size: raise error("unpack_from requires a buffer of at least %d bytes" % (size,)) return unpack(fmt, data) def _clearcache(): "Clear the internal cache." # No cache in this implementation ================================================ FILE: third_party/pypy/binascii.py ================================================ """A pure Python implementation of binascii. Rather slow and buggy in corner cases. PyPy provides an RPython version too. """ class Error(Exception): pass class Done(Exception): pass class Incomplete(Exception): pass def a2b_uu(s): if not s: return '' length = (ord(s[0]) - 0x20) % 64 def quadruplets_gen(s): while s: try: yield ord(s[0]), ord(s[1]), ord(s[2]), ord(s[3]) except IndexError: s += ' ' yield ord(s[0]), ord(s[1]), ord(s[2]), ord(s[3]) return s = s[4:] try: result = [''.join( [chr((A - 0x20) << 2 | (((B - 0x20) >> 4) & 0x3)), chr(((B - 0x20) & 0xf) << 4 | (((C - 0x20) >> 2) & 0xf)), chr(((C - 0x20) & 0x3) << 6 | ((D - 0x20) & 0x3f)) ]) for A, B, C, D in quadruplets_gen(s[1:].rstrip())] except ValueError: raise Error('Illegal char') result = ''.join(result) trailingdata = result[length:] # if trailingdata.strip('\x00'): # raise Error('Trailing garbage') result = result[:length] if len(result) < length: result += ((length - len(result)) * '\x00') return result def b2a_uu(s): length = len(s) if length > 45: raise Error('At most 45 bytes at once') def triples_gen(s): while s: try: yield ord(s[0]), ord(s[1]), ord(s[2]) except IndexError: s += '\0\0' yield ord(s[0]), ord(s[1]), ord(s[2]) return s = s[3:] result = [''.join( [chr(0x20 + (( A >> 2 ) & 0x3F)), chr(0x20 + (((A << 4) | ((B >> 4) & 0xF)) & 0x3F)), chr(0x20 + (((B << 2) | ((C >> 6) & 0x3)) & 0x3F)), chr(0x20 + (( C ) & 0x3F))]) for A, B, C in triples_gen(s)] return chr(ord(' ') + (length & 077)) + ''.join(result) + '\n' table_a2b_base64 = { 'A': 0, 'B': 1, 'C': 2, 'D': 3, 'E': 4, 'F': 5, 'G': 6, 'H': 7, 'I': 8, 'J': 9, 'K': 10, 'L': 11, 'M': 12, 'N': 13, 'O': 14, 'P': 15, 'Q': 16, 'R': 17, 'S': 18, 'T': 19, 'U': 20, 'V': 21, 'W': 22, 'X': 23, 'Y': 24, 'Z': 25, 'a': 26, 'b': 27, 'c': 28, 'd': 29, 'e': 30, 'f': 31, 'g': 32, 'h': 33, 'i': 34, 'j': 35, 'k': 36, 'l': 37, 'm': 38, 'n': 39, 'o': 40, 'p': 41, 'q': 42, 'r': 43, 's': 44, 't': 45, 'u': 46, 'v': 47, 'w': 48, 'x': 49, 'y': 50, 'z': 51, '0': 52, '1': 53, '2': 54, '3': 55, '4': 56, '5': 57, '6': 58, '7': 59, '8': 60, '9': 61, '+': 62, '/': 63, '=': 0, } def a2b_base64(s): if not isinstance(s, (str, unicode)): raise TypeError("expected string or unicode, got %r" % (s,)) s = s.rstrip() # clean out all invalid characters, this also strips the final '=' padding # check for correct padding def next_valid_char(s, pos): for i in range(pos + 1, len(s)): c = s[i] if c < '\x7f': try: table_a2b_base64[c] return c except KeyError: pass return None quad_pos = 0 leftbits = 0 leftchar = 0 res = [] for i, c in enumerate(s): if c > '\x7f' or c == '\n' or c == '\r' or c == ' ': continue if c == '=': if quad_pos < 2 or (quad_pos == 2 and next_valid_char(s, i) != '='): continue else: leftbits = 0 break try: next_c = table_a2b_base64[c] except KeyError: continue quad_pos = (quad_pos + 1) & 0x03 leftchar = (leftchar << 6) | next_c leftbits += 6 if leftbits >= 8: leftbits -= 8 res.append((leftchar >> leftbits & 0xff)) leftchar &= ((1 << leftbits) - 1) if leftbits != 0: raise Error('Incorrect padding') return ''.join([chr(i) for i in res]) table_b2a_base64 = \ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" def b2a_base64(s): length = len(s) final_length = length % 3 def triples_gen(s): while s: try: yield ord(s[0]), ord(s[1]), ord(s[2]) except IndexError: s += '\0\0' yield ord(s[0]), ord(s[1]), ord(s[2]) return s = s[3:] a = triples_gen(s[ :length - final_length]) result = [''.join( [table_b2a_base64[( A >> 2 ) & 0x3F], table_b2a_base64[((A << 4) | ((B >> 4) & 0xF)) & 0x3F], table_b2a_base64[((B << 2) | ((C >> 6) & 0x3)) & 0x3F], table_b2a_base64[( C ) & 0x3F]]) for A, B, C in a] final = s[length - final_length:] if final_length == 0: snippet = '' elif final_length == 1: a = ord(final[0]) snippet = table_b2a_base64[(a >> 2 ) & 0x3F] + \ table_b2a_base64[(a << 4 ) & 0x3F] + '==' else: a = ord(final[0]) b = ord(final[1]) snippet = table_b2a_base64[(a >> 2) & 0x3F] + \ table_b2a_base64[((a << 4) | (b >> 4) & 0xF) & 0x3F] + \ table_b2a_base64[(b << 2) & 0x3F] + '=' return ''.join(result) + snippet + '\n' def a2b_qp(s, header=False): inp = 0 odata = [] while inp < len(s): if s[inp] == '=': inp += 1 if inp >= len(s): break # Soft line breaks if (s[inp] == '\n') or (s[inp] == '\r'): if s[inp] != '\n': while inp < len(s) and s[inp] != '\n': inp += 1 if inp < len(s): inp += 1 elif s[inp] == '=': # broken case from broken python qp odata.append('=') inp += 1 elif s[inp] in hex_numbers and s[inp + 1] in hex_numbers: ch = chr(int(s[inp:inp+2], 16)) inp += 2 odata.append(ch) else: odata.append('=') elif header and s[inp] == '_': odata.append(' ') inp += 1 else: odata.append(s[inp]) inp += 1 return ''.join(odata) def b2a_qp(data, quotetabs=False, istext=True, header=False): """quotetabs=True means that tab and space characters are always quoted. istext=False means that \r and \n are treated as regular characters header=True encodes space characters with '_' and requires real '_' characters to be quoted. """ MAXLINESIZE = 76 # See if this string is using CRLF line ends lf = data.find('\n') crlf = lf > 0 and data[lf-1] == '\r' inp = 0 linelen = 0 odata = [] while inp < len(data): c = data[inp] if (c > '~' or c == '=' or (header and c == '_') or (c == '.' and linelen == 0 and (inp+1 == len(data) or data[inp+1] == '\n' or data[inp+1] == '\r')) or (not istext and (c == '\r' or c == '\n')) or ((c == '\t' or c == ' ') and (inp + 1 == len(data))) or (c <= ' ' and c != '\r' and c != '\n' and (quotetabs or (not quotetabs and (c != '\t' and c != ' '))))): linelen += 3 if linelen >= MAXLINESIZE: odata.append('=') if crlf: odata.append('\r') odata.append('\n') linelen = 3 odata.append('=' + two_hex_digits(ord(c))) inp += 1 else: if (istext and (c == '\n' or (inp+1 < len(data) and c == '\r' and data[inp+1] == '\n'))): linelen = 0 # Protect against whitespace on end of line if (len(odata) > 0 and (odata[-1] == ' ' or odata[-1] == '\t')): ch = ord(odata[-1]) odata[-1] = '=' odata.append(two_hex_digits(ch)) if crlf: odata.append('\r') odata.append('\n') if c == '\r': inp += 2 else: inp += 1 else: if (inp + 1 < len(data) and data[inp+1] != '\n' and (linelen + 1) >= MAXLINESIZE): odata.append('=') if crlf: odata.append('\r') odata.append('\n') linelen = 0 linelen += 1 if header and c == ' ': c = '_' odata.append(c) inp += 1 return ''.join(odata) hex_numbers = '0123456789ABCDEF' def hex(n): if n == 0: return '0' if n < 0: n = -n sign = '-' else: sign = '' arr = [] def hex_gen(n): """ Yield a nibble at a time. """ while n: yield n % 0x10 n = n / 0x10 for nibble in hex_gen(n): arr = [hex_numbers[nibble]] + arr return sign + ''.join(arr) def two_hex_digits(n): return hex_numbers[n / 0x10] + hex_numbers[n % 0x10] def strhex_to_int(s): i = 0 for c in s: i = i * 0x10 + hex_numbers.index(c) return i hqx_encoding = '!"#$%&\'()*+,-012345689@ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr' DONE = 0x7f SKIP = 0x7e FAIL = 0x7d table_a2b_hqx = [ #^@ ^A ^B ^C ^D ^E ^F ^G FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, #\b \t \n ^K ^L \r ^N ^O FAIL, FAIL, SKIP, FAIL, FAIL, SKIP, FAIL, FAIL, #^P ^Q ^R ^S ^T ^U ^V ^W FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, #^X ^Y ^Z ^[ ^\ ^] ^^ ^_ FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, # ! " # $ % & ' FAIL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, #( ) * + , - . / 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, FAIL, FAIL, #0 1 2 3 4 5 6 7 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, FAIL, #8 9 : ; < = > ? 0x14, 0x15, DONE, FAIL, FAIL, FAIL, FAIL, FAIL, #@ A B C D E F G 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, #H I J K L M N O 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, FAIL, #P Q R S T U V W 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, FAIL, #X Y Z [ \ ] ^ _ 0x2C, 0x2D, 0x2E, 0x2F, FAIL, FAIL, FAIL, FAIL, #` a b c d e f g 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, FAIL, #h i j k l m n o 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, FAIL, FAIL, #p q r s t u v w 0x3D, 0x3E, 0x3F, FAIL, FAIL, FAIL, FAIL, FAIL, #x y z { | } ~ ^? FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, ] def a2b_hqx(s): result = [] def quadruples_gen(s): t = [] for c in s: res = table_a2b_hqx[ord(c)] if res == SKIP: continue elif res == FAIL: raise Error('Illegal character') elif res == DONE: yield t raise Done else: t.append(res) if len(t) == 4: yield t t = [] yield t done = 0 try: for snippet in quadruples_gen(s): length = len(snippet) if length == 4: result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4))) result.append(chr(((snippet[1] & 0x0f) << 4) | (snippet[2] >> 2))) result.append(chr(((snippet[2] & 0x03) << 6) | (snippet[3]))) elif length == 3: result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4))) result.append(chr(((snippet[1] & 0x0f) << 4) | (snippet[2] >> 2))) elif length == 2: result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4))) except Done: done = 1 except Error: raise return (''.join(result), done) def b2a_hqx(s): result =[] def triples_gen(s): while s: try: yield ord(s[0]), ord(s[1]), ord(s[2]) except IndexError: yield tuple([ord(c) for c in s]) s = s[3:] for snippet in triples_gen(s): length = len(snippet) if length == 3: result.append( hqx_encoding[(snippet[0] & 0xfc) >> 2]) result.append(hqx_encoding[ ((snippet[0] & 0x03) << 4) | ((snippet[1] & 0xf0) >> 4)]) result.append(hqx_encoding[ (snippet[1] & 0x0f) << 2 | ((snippet[2] & 0xc0) >> 6)]) result.append(hqx_encoding[snippet[2] & 0x3f]) elif length == 2: result.append( hqx_encoding[(snippet[0] & 0xfc) >> 2]) result.append(hqx_encoding[ ((snippet[0] & 0x03) << 4) | ((snippet[1] & 0xf0) >> 4)]) result.append(hqx_encoding[ (snippet[1] & 0x0f) << 2]) elif length == 1: result.append( hqx_encoding[(snippet[0] & 0xfc) >> 2]) result.append(hqx_encoding[ ((snippet[0] & 0x03) << 4)]) return ''.join(result) crctab_hqx = [ 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0, ] def crc_hqx(s, crc): for c in s: crc = ((crc << 8) & 0xff00) ^ crctab_hqx[((crc >> 8) & 0xff) ^ ord(c)] return crc def rlecode_hqx(s): """ Run length encoding for binhex4. The CPython implementation does not do run length encoding of \x90 characters. This implementation does. """ if not s: return '' result = [] prev = s[0] count = 1 # Add a dummy character to get the loop to go one extra round. # The dummy must be different from the last character of s. # In the same step we remove the first character, which has # already been stored in prev. if s[-1] == '!': s = s[1:] + '?' else: s = s[1:] + '!' for c in s: if c == prev and count < 255: count += 1 else: if count == 1: if prev != '\x90': result.append(prev) else: result += ['\x90', '\x00'] elif count < 4: if prev != '\x90': result += [prev] * count else: result += ['\x90', '\x00'] * count else: if prev != '\x90': result += [prev, '\x90', chr(count)] else: result += ['\x90', '\x00', '\x90', chr(count)] count = 1 prev = c return ''.join(result) def rledecode_hqx(s): s = s.split('\x90') result = [s[0]] prev = s[0] for snippet in s[1:]: count = ord(snippet[0]) if count > 0: result.append(prev[-1] * (count-1)) prev = snippet else: result. append('\x90') prev = '\x90' result.append(snippet[1:]) return ''.join(result) crc_32_tab = [ 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L, 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL, 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L, 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L, 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L, 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL, 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L, 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL, 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L, 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L, 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L, 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL, 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL, 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L, 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL, 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L, 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L, 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L, 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL, 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L, 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL, 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L, 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L, 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL, 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL, 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L, 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL, 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L, 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL, 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L, 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL, 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L, 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L, 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL, 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L, 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L, 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L, 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L, 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L, 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L, 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL, 0x2d02ef8dL ] def crc32(s, crc=0): result = 0 crc = ~long(crc) & 0xffffffffL for c in s: crc = crc_32_tab[(crc ^ long(ord(c))) & 0xffL] ^ (crc >> 8) #/* Note: (crc >> 8) MUST zero fill on left result = crc ^ 0xffffffffL if result > (1 << 31): result = ((result + (1<<31)) % (1<<32)) - (1<<31) return result def b2a_hex(s): result = [] for char in s: c = (ord(char) >> 4) & 0xf if c > 9: c = c + ord('a') - 10 else: c = c + ord('0') result.append(chr(c)) c = ord(char) & 0xf if c > 9: c = c + ord('a') - 10 else: c = c + ord('0') result.append(chr(c)) return ''.join(result) hexlify = b2a_hex table_hex = [ -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1, -1,-1,-1,-1, -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1 ] def a2b_hex(t): result = [] def pairs_gen(s): while s: try: yield table_hex[ord(s[0])], table_hex[ord(s[1])] except IndexError: if len(s): raise TypeError('Odd-length string') return s = s[2:] for a, b in pairs_gen(t): if a < 0 or b < 0: raise TypeError('Non-hexadecimal digit found') result.append(chr((a << 4) + b)) return ''.join(result) unhexlify = a2b_hex ================================================ FILE: third_party/pypy/datetime.py ================================================ """Concrete date/time and related types -- prototype implemented in Python. See http://www.zope.org/Members/fdrake/DateTimeWiki/FrontPage See also http://dir.yahoo.com/Reference/calendars/ For a primer on DST, including many current DST rules, see http://webexhibits.org/daylightsaving/ For more about DST than you ever wanted to know, see ftp://elsie.nci.nih.gov/pub/ Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm This was originally copied from the sandbox of the CPython CVS repository. Thanks to Tim Peters for suggesting using it. """ # from __future__ import division import time as _time import math as _math # import struct as _struct import _struct def divmod(x, y): x, y = int(x), int(y) return x / y, x % y _SENTINEL = object() def _cmp(x, y): return 0 if x == y else 1 if x > y else -1 def _round(x): return int(_math.floor(x + 0.5) if x >= 0.0 else _math.ceil(x - 0.5)) MINYEAR = 1 MAXYEAR = 9999 _MINYEARFMT = 1900 _MAX_DELTA_DAYS = 999999999 # Utility functions, adapted from Python's Demo/classes/Dates.py, which # also assumes the current Gregorian calendar indefinitely extended in # both directions. Difference: Dates.py calls January 1 of year 0 day # number 1. The code here calls January 1 of year 1 day number 1. This is # to match the definition of the "proleptic Gregorian" calendar in Dershowitz # and Reingold's "Calendrical Calculations", where it's the base calendar # for all computations. See the book for algorithms for converting between # proleptic Gregorian ordinals and many other calendar systems. _DAYS_IN_MONTH = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] _DAYS_BEFORE_MONTH = [-1] dbm = 0 for dim in _DAYS_IN_MONTH[1:]: _DAYS_BEFORE_MONTH.append(dbm) dbm += dim del dbm, dim def _is_leap(year): "year -> 1 if leap year, else 0." return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0) def _days_before_year(year): "year -> number of days before January 1st of year." y = year - 1 return y*365 + y//4 - y//100 + y//400 def _days_in_month(year, month): "year, month -> number of days in that month in that year." assert 1 <= month <= 12, month if month == 2 and _is_leap(year): return 29 return _DAYS_IN_MONTH[month] def _days_before_month(year, month): "year, month -> number of days in year preceding first day of month." assert 1 <= month <= 12, 'month must be in 1..12' return _DAYS_BEFORE_MONTH[month] + (month > 2 and _is_leap(year)) def _ymd2ord(year, month, day): "year, month, day -> ordinal, considering 01-Jan-0001 as day 1." assert 1 <= month <= 12, 'month must be in 1..12' dim = _days_in_month(year, month) assert 1 <= day <= dim, ('day must be in 1..%d' % dim) return (_days_before_year(year) + _days_before_month(year, month) + day) _DI400Y = _days_before_year(401) # number of days in 400 years _DI100Y = _days_before_year(101) # " " " " 100 " _DI4Y = _days_before_year(5) # " " " " 4 " # A 4-year cycle has an extra leap day over what we'd get from pasting # together 4 single years. assert _DI4Y == 4 * 365 + 1 # Similarly, a 400-year cycle has an extra leap day over what we'd get from # pasting together 4 100-year cycles. assert _DI400Y == 4 * _DI100Y + 1 # OTOH, a 100-year cycle has one fewer leap day than we'd get from # pasting together 25 4-year cycles. assert _DI100Y == 25 * _DI4Y - 1 _US_PER_US = 1 _US_PER_MS = 1000 _US_PER_SECOND = 1000000 _US_PER_MINUTE = 60000000 _SECONDS_PER_DAY = 24 * 3600 _US_PER_HOUR = 3600000000 _US_PER_DAY = 86400000000 _US_PER_WEEK = 604800000000 def _ord2ymd(n): "ordinal -> (year, month, day), considering 01-Jan-0001 as day 1." # n is a 1-based index, starting at 1-Jan-1. The pattern of leap years # repeats exactly every 400 years. The basic strategy is to find the # closest 400-year boundary at or before n, then work with the offset # from that boundary to n. Life is much clearer if we subtract 1 from # n first -- then the values of n at 400-year boundaries are exactly # those divisible by _DI400Y: # # D M Y n n-1 # -- --- ---- ---------- ---------------- # 31 Dec -400 -_DI400Y -_DI400Y -1 # 1 Jan -399 -_DI400Y +1 -_DI400Y 400-year boundary # ... # 30 Dec 000 -1 -2 # 31 Dec 000 0 -1 # 1 Jan 001 1 0 400-year boundary # 2 Jan 001 2 1 # 3 Jan 001 3 2 # ... # 31 Dec 400 _DI400Y _DI400Y -1 # 1 Jan 401 _DI400Y +1 _DI400Y 400-year boundary n -= 1 n400, n = divmod(n, _DI400Y) year = n400 * 400 + 1 # ..., -399, 1, 401, ... # Now n is the (non-negative) offset, in days, from January 1 of year, to # the desired date. Now compute how many 100-year cycles precede n. # Note that it's possible for n100 to equal 4! In that case 4 full # 100-year cycles precede the desired day, which implies the desired # day is December 31 at the end of a 400-year cycle. n100, n = divmod(n, _DI100Y) # Now compute how many 4-year cycles precede it. n4, n = divmod(n, _DI4Y) # And now how many single years. Again n1 can be 4, and again meaning # that the desired day is December 31 at the end of the 4-year cycle. n1, n = divmod(n, 365) year += n100 * 100 + n4 * 4 + n1 if n1 == 4 or n100 == 4: assert n == 0 return year-1, 12, 31 # Now the year is correct, and n is the offset from January 1. We find # the month via an estimate that's either exact or one too large. leapyear = n1 == 3 and (n4 != 24 or n100 == 3) assert leapyear == _is_leap(year) month = (n + 50) >> 5 preceding = _DAYS_BEFORE_MONTH[month] + (month > 2 and leapyear) if preceding > n: # estimate is too large month -= 1 preceding -= _DAYS_IN_MONTH[month] + (month == 2 and leapyear) n -= preceding assert 0 <= n < _days_in_month(year, month) # Now the year and month are correct, and n is the offset from the # start of that month: we're done! return year, month, n+1 # Month and day names. For localized versions, see the calendar module. _MONTHNAMES = [None, "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] _DAYNAMES = [None, "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] def _build_struct_time(y, m, d, hh, mm, ss, dstflag): wday = (_ymd2ord(y, m, d) + 6) % 7 dnum = _days_before_month(y, m) + d return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) def _format_time(hh, mm, ss, us): # Skip trailing microseconds when us==0. result = "%02d:%02d:%02d" % (hh, mm, ss) if us: result += ".%06d" % us return result # Correctly substitute for %z and %Z escapes in strftime formats. # def _wrap_strftime(object, format, timetuple): # year = timetuple[0] # if year < _MINYEARFMT: # raise ValueError("year=%d is before %d; the datetime strftime() " # "methods require year >= %d" % # (year, _MINYEARFMT, _MINYEARFMT)) # # Don't call utcoffset() or tzname() unless actually needed. # freplace = None # the string to use for %f # zreplace = None # the string to use for %z # Zreplace = None # the string to use for %Z # # Scan format for %z and %Z escapes, replacing as needed. # newformat = [] # push = newformat.append # i, n = 0, len(format) # while i < n: # ch = format[i] # i += 1 # if ch == '%': # if i < n: # ch = format[i] # i += 1 # if ch == 'f': # if freplace is None: # freplace = '%06d' % getattr(object, # 'microsecond', 0) # newformat.append(freplace) # elif ch == 'z': # if zreplace is None: # zreplace = "" # if hasattr(object, "_utcoffset"): # offset = object._utcoffset() # if offset is not None: # sign = '+' # if offset < 0: # offset = -offset # sign = '-' # h, m = divmod(offset, 60) # zreplace = '%c%02d%02d' % (sign, h, m) # assert '%' not in zreplace # newformat.append(zreplace) # elif ch == 'Z': # if Zreplace is None: # Zreplace = "" # if hasattr(object, "tzname"): # s = object.tzname() # if s is not None: # # strftime is going to have at this: escape % # Zreplace = s.replace('%', '%%') # newformat.append(Zreplace) # else: # push('%') # push(ch) # else: # push('%') # else: # push(ch) # newformat = "".join(newformat) # return _time.strftime(newformat, timetuple) # Just raise TypeError if the arg isn't None or a string. def _check_tzname(name): if name is not None and not isinstance(name, str): raise TypeError("tzinfo.tzname() must return None or string, " "not '%s'" % type(name)) # name is the offset-producing method, "utcoffset" or "dst". # offset is what it returned. # If offset isn't None or timedelta, raises TypeError. # If offset is None, returns None. # Else offset is checked for being in range, and a whole # of minutes. # If it is, its integer value is returned. Else ValueError is raised. def _check_utc_offset(name, offset): assert name in ("utcoffset", "dst") if offset is None: return if not isinstance(offset, timedelta): raise TypeError("tzinfo.%s() must return None " "or timedelta, not '%s'" % (name, type(offset))) days = offset.days if days < -1 or days > 0: offset = 1440 # trigger out-of-range else: seconds = days * 86400 + offset.seconds minutes, seconds = divmod(seconds, 60) if seconds or offset.microseconds: raise ValueError("tzinfo.%s() must return a whole number " "of minutes" % name) offset = minutes if not -1440 < offset < 1440: raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset)) return offset def _check_int_field(value): if isinstance(value, int): return int(value) if not isinstance(value, float): try: value = value.__int__() except AttributeError: pass else: if isinstance(value, int): return int(value) elif isinstance(value, long): return int(long(value)) raise TypeError('__int__ method should return an integer') raise TypeError('an integer is required') raise TypeError('integer argument expected, got float') def _check_date_fields(year, month, day): year = _check_int_field(year) month = _check_int_field(month) day = _check_int_field(day) if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: raise ValueError('month must be in 1..12', month) dim = _days_in_month(year, month) if not 1 <= day <= dim: raise ValueError('day must be in 1..%d' % dim, day) return year, month, day def _check_time_fields(hour, minute, second, microsecond): hour = _check_int_field(hour) minute = _check_int_field(minute) second = _check_int_field(second) microsecond = _check_int_field(microsecond) if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: raise ValueError('minute must be in 0..59', minute) if not 0 <= second <= 59: raise ValueError('second must be in 0..59', second) if not 0 <= microsecond <= 999999: raise ValueError('microsecond must be in 0..999999', microsecond) return hour, minute, second, microsecond def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): raise TypeError("tzinfo argument must be None or of a tzinfo subclass") # Notes on comparison: In general, datetime module comparison operators raise # TypeError when they don't know how to do a comparison themself. If they # returned NotImplemented instead, comparison could (silently) fall back to # the default compare-objects-by-comparing-their-memory-addresses strategy, # and that's not helpful. There are two exceptions: # # 1. For date and datetime, if the other object has a "timetuple" attr, # NotImplemented is returned. This is a hook to allow other kinds of # datetime-like objects a chance to intercept the comparison. # # 2. Else __eq__ and __ne__ return False and True, respectively. This is # so opertaions like # # x == y # x != y # x in sequence # x not in sequence # dict[x] = y # # don't raise annoying TypeErrors just because a datetime object # is part of a heterogeneous collection. If there's no known way to # compare X to a datetime, saying they're not equal is reasonable. def _cmperror(x, y): raise TypeError("can't compare '%s' to '%s'" % ( type(x).__name__, type(y).__name__)) def _normalize_pair(hi, lo, factor): if not 0 <= lo <= factor-1: inc, lo = divmod(lo, factor) hi += inc return hi, lo def _normalize_datetime(y, m, d, hh, mm, ss, us, ignore_overflow=False): # Normalize all the inputs, and store the normalized values. ss, us = _normalize_pair(ss, us, 1000000) mm, ss = _normalize_pair(mm, ss, 60) hh, mm = _normalize_pair(hh, mm, 60) d, hh = _normalize_pair(d, hh, 24) y, m, d = _normalize_date(y, m, d, ignore_overflow) return y, m, d, hh, mm, ss, us def _normalize_date(year, month, day, ignore_overflow=False): # That was easy. Now it gets muddy: the proper range for day # can't be determined without knowing the correct month and year, # but if day is, e.g., plus or minus a million, the current month # and year values make no sense (and may also be out of bounds # themselves). # Saying 12 months == 1 year should be non-controversial. if not 1 <= month <= 12: year, month = _normalize_pair(year, month-1, 12) month += 1 assert 1 <= month <= 12 # Now only day can be out of bounds (year may also be out of bounds # for a datetime object, but we don't care about that here). # If day is out of bounds, what to do is arguable, but at least the # method here is principled and explainable. dim = _days_in_month(year, month) if not 1 <= day <= dim: # Move day-1 days from the first of the month. First try to # get off cheap if we're only one day out of range (adjustments # for timezone alone can't be worse than that). if day == 0: # move back a day month -= 1 if month > 0: day = _days_in_month(year, month) else: year, month, day = year-1, 12, 31 elif day == dim + 1: # move forward a day month += 1 day = 1 if month > 12: month = 1 year += 1 else: ordinal = _ymd2ord(year, month, 1) + (day - 1) year, month, day = _ord2ymd(ordinal) if not ignore_overflow and not MINYEAR <= year <= MAXYEAR: raise OverflowError("date value out of range") return year, month, day def _accum(tag, sofar, num, factor, leftover): if isinstance(num, (int, long)): prod = num * factor rsum = sofar + prod return rsum, leftover if isinstance(num, float): fracpart, intpart = _math.modf(num) prod = int(intpart) * factor rsum = sofar + prod if fracpart == 0.0: return rsum, leftover assert isinstance(factor, (int, long)) fracpart, intpart = _math.modf(factor * fracpart) rsum += int(intpart) return rsum, leftover + fracpart raise TypeError("unsupported type for timedelta %s component: %s" % (tag, type(num))) class timedelta(object): """Represent the difference between two datetime objects. Supported operators: - add, subtract timedelta - unary plus, minus, abs - compare to timedelta - multiply, divide by int/long In addition, datetime supports subtraction of two datetime objects returning a timedelta, and addition or subtraction of a datetime and a timedelta giving a datetime. Representation: (days, seconds, microseconds). Why? Because I felt like it. """ __slots__ = '_days', '_seconds', '_microseconds', '_hashcode' def __new__(cls, days=_SENTINEL, seconds=_SENTINEL, microseconds=_SENTINEL, milliseconds=_SENTINEL, minutes=_SENTINEL, hours=_SENTINEL, weeks=_SENTINEL): x = 0 leftover = 0.0 if microseconds is not _SENTINEL: x, leftover = _accum("microseconds", x, microseconds, _US_PER_US, leftover) if milliseconds is not _SENTINEL: x, leftover = _accum("milliseconds", x, milliseconds, _US_PER_MS, leftover) if seconds is not _SENTINEL: x, leftover = _accum("seconds", x, seconds, _US_PER_SECOND, leftover) if minutes is not _SENTINEL: x, leftover = _accum("minutes", x, minutes, _US_PER_MINUTE, leftover) if hours is not _SENTINEL: x, leftover = _accum("hours", x, hours, _US_PER_HOUR, leftover) if days is not _SENTINEL: x, leftover = _accum("days", x, days, _US_PER_DAY, leftover) if weeks is not _SENTINEL: x, leftover = _accum("weeks", x, weeks, _US_PER_WEEK, leftover) if leftover != 0.0: x += _round(leftover) return cls._from_microseconds(x) @classmethod def _from_microseconds(cls, us): s, us = divmod(us, _US_PER_SECOND) d, s = divmod(s, _SECONDS_PER_DAY) return cls._create(d, s, us, False) @classmethod def _create(cls, d, s, us, normalize): if normalize: s, us = _normalize_pair(s, us, 1000000) d, s = _normalize_pair(d, s, 24*3600) if not -_MAX_DELTA_DAYS <= d <= _MAX_DELTA_DAYS: raise OverflowError("days=%d; must have magnitude <= %d" % (d, _MAX_DELTA_DAYS)) self = object.__new__(cls) self._days = d self._seconds = s self._microseconds = us self._hashcode = -1 return self def _to_microseconds(self): return ((self._days * _SECONDS_PER_DAY + self._seconds) * _US_PER_SECOND + self._microseconds) def __repr__(self): module = "datetime." if self.__class__ is timedelta else "" if self._microseconds: return "%s(%d, %d, %d)" % (module + self.__class__.__name__, self._days, self._seconds, self._microseconds) if self._seconds: return "%s(%d, %d)" % (module + self.__class__.__name__, self._days, self._seconds) return "%s(%d)" % (module + self.__class__.__name__, self._days) def __str__(self): mm, ss = divmod(self._seconds, 60) hh, mm = divmod(mm, 60) s = "%d:%02d:%02d" % (hh, mm, ss) if self._days: def plural(n): return n, abs(n) != 1 and "s" or "" s = ("%d day%s, " % plural(self._days)) + s if self._microseconds: s = s + ".%06d" % self._microseconds return s def total_seconds(self): """Total seconds in the duration.""" # return self._to_microseconds() / 10**6 return float(self._to_microseconds()) / float(10**6) # Read-only field accessors @property def days(self): """days""" return self._days @property def seconds(self): """seconds""" return self._seconds @property def microseconds(self): """microseconds""" return self._microseconds def __add__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta return timedelta._create(self._days + other._days, self._seconds + other._seconds, self._microseconds + other._microseconds, True) return NotImplemented def __sub__(self, other): if isinstance(other, timedelta): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta return timedelta._create(self._days - other._days, self._seconds - other._seconds, self._microseconds - other._microseconds, True) return NotImplemented def __neg__(self): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta return timedelta._create(-self._days, -self._seconds, -self._microseconds, True) def __pos__(self): # for CPython compatibility, we cannot use # our __class__ here, but need a real timedelta return timedelta._create(self._days, self._seconds, self._microseconds, False) def __abs__(self): if self._days < 0: return -self else: return self def __mul__(self, other): if not isinstance(other, (int, long)): return NotImplemented usec = self._to_microseconds() return timedelta._from_microseconds(usec * other) __rmul__ = __mul__ def __div__(self, other): if not isinstance(other, (int, long)): return NotImplemented usec = self._to_microseconds() # return timedelta._from_microseconds(usec // other) return timedelta._from_microseconds(int(usec) / int(other)) __floordiv__ = __div__ # Comparisons of timedelta objects with other. def __eq__(self, other): if isinstance(other, timedelta): return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, timedelta): return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, timedelta): return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, timedelta): return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, timedelta): return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, timedelta): return self._cmp(other) > 0 else: _cmperror(self, other) def _cmp(self, other): assert isinstance(other, timedelta) return _cmp(self._getstate(), other._getstate()) def __hash__(self): if self._hashcode == -1: self._hashcode = hash(self._getstate()) return self._hashcode def __nonzero__(self): return (self._days != 0 or self._seconds != 0 or self._microseconds != 0) # Pickle support. def _getstate(self): return (self._days, self._seconds, self._microseconds) def __reduce__(self): return (self.__class__, self._getstate()) timedelta.min = timedelta(-_MAX_DELTA_DAYS) timedelta.max = timedelta(_MAX_DELTA_DAYS, 24*3600-1, 1000000-1) timedelta.resolution = timedelta(microseconds=1) class date(object): """Concrete date type. Constructors: __new__() fromtimestamp() today() fromordinal() Operators: __repr__, __str__ __cmp__, __hash__ __add__, __radd__, __sub__ (add/radd only with timedelta arg) Methods: timetuple() toordinal() weekday() isoweekday(), isocalendar(), isoformat() ctime() strftime() Properties (readonly): year, month, day """ __slots__ = '_year', '_month', '_day', '_hashcode' def __new__(cls, year, month=None, day=None): """Constructor. Arguments: year, month, day (required, base 1) """ # if month is None and isinstance(year, bytes) and len(year) == 4 and \ # 1 <= ord(year[2]) <= 12: # # Pickle support # self = object.__new__(cls) # self.__setstate(year) # self._hashcode = -1 # return self year, month, day = _check_date_fields(year, month, day) self = object.__new__(cls) self._year = year self._month = month self._day = day self._hashcode = -1 return self # Additional constructors @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) @classmethod def today(cls): "Construct a date from time.time()." t = _time.time() return cls.fromtimestamp(t) @classmethod def fromordinal(cls, n): """Contruct a date from a proleptic Gregorian ordinal. January 1 of year 1 is day 1. Only the year, month and day are non-zero in the result. """ y, m, d = _ord2ymd(n) return cls(y, m, d) # Conversions to string def __repr__(self): """Convert to formal string, for repr(). >>> dt = datetime(2010, 1, 1) >>> repr(dt) 'datetime.datetime(2010, 1, 1, 0, 0)' >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) >>> repr(dt) 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' """ module = "datetime." if self.__class__ is date else "" return "%s(%d, %d, %d)" % (module + self.__class__.__name__, self._year, self._month, self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because # strftime("%c", ...) is locale specific. def ctime(self): "Return ctime() style string." weekday = self.toordinal() % 7 or 7 return "%s %s %2d 00:00:00 %04d" % ( _DAYNAMES[weekday], _MONTHNAMES[self._month], self._day, self._year) # def strftime(self, format): # "Format using strftime()." # return _wrap_strftime(self, format, self.timetuple()) def __format__(self, fmt): if not isinstance(fmt, (str, unicode)): raise ValueError("__format__ expects str or unicode, not %s" % fmt.__class__.__name__) if len(fmt) != 0: return self.strftime(fmt) return str(self) def isoformat(self): """Return the date formatted according to ISO. This is 'YYYY-MM-DD'. References: - http://www.w3.org/TR/NOTE-datetime - http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ # return "%04d-%02d-%02d" % (self._year, self._month, self._day) return "%s-%s-%s" % (str(self._year).zfill(4), str(self._month).zfill(2), str(self._day).zfill(2)) __str__ = isoformat # Read-only field accessors @property def year(self): """year (1-9999)""" return self._year @property def month(self): """month (1-12)""" return self._month @property def day(self): """day (1-31)""" return self._day # Standard conversions, __cmp__, __hash__ (and helpers) def timetuple(self): "Return local time tuple compatible with time.localtime()." return _build_struct_time(self._year, self._month, self._day, 0, 0, 0, -1) def toordinal(self): """Return proleptic Gregorian ordinal for the year, month and day. January 1 of year 1 is day 1. Only the year, month and day values contribute to the result. """ return _ymd2ord(self._year, self._month, self._day) def replace(self, year=None, month=None, day=None): """Return a new date with new values for the specified fields.""" if year is None: year = self._year if month is None: month = self._month if day is None: day = self._day return date.__new__(type(self), year, month, day) # Comparisons of date objects with other. def __eq__(self, other): if isinstance(other, date): return self._cmp(other) == 0 elif hasattr(other, "timetuple"): return NotImplemented else: return False def __ne__(self, other): if isinstance(other, date): return self._cmp(other) != 0 elif hasattr(other, "timetuple"): return NotImplemented else: return True def __le__(self, other): if isinstance(other, date): return self._cmp(other) <= 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, date): return self._cmp(other) < 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, date): return self._cmp(other) >= 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, date): return self._cmp(other) > 0 elif hasattr(other, "timetuple"): return NotImplemented else: _cmperror(self, other) def _cmp(self, other): assert isinstance(other, date) y, m, d = self._year, self._month, self._day y2, m2, d2 = other._year, other._month, other._day return _cmp((y, m, d), (y2, m2, d2)) def __hash__(self): "Hash." if self._hashcode == -1: self._hashcode = hash(self._getstate()) return self._hashcode # Computations def _add_timedelta(self, other, factor): y, m, d = _normalize_date( self._year, self._month, self._day + other.days * factor) return date(y, m, d) def __add__(self, other): "Add a date to a timedelta." if isinstance(other, timedelta): return self._add_timedelta(other, 1) return NotImplemented __radd__ = __add__ def __sub__(self, other): """Subtract two dates, or a date and a timedelta.""" if isinstance(other, date): days1 = self.toordinal() days2 = other.toordinal() return timedelta._create(days1 - days2, 0, 0, False) if isinstance(other, timedelta): return self._add_timedelta(other, -1) return NotImplemented def weekday(self): "Return day of the week, where Monday == 0 ... Sunday == 6." return (self.toordinal() + 6) % 7 # Day-of-the-week and week-of-the-year, according to ISO def isoweekday(self): "Return day of the week, where Monday == 1 ... Sunday == 7." # 1-Jan-0001 is a Monday return self.toordinal() % 7 or 7 def isocalendar(self): """Return a 3-tuple containing ISO year, week number, and weekday. The first ISO week of the year is the (Mon-Sun) week containing the year's first Thursday; everything else derives from that. The first week is 1; Monday is 1 ... Sunday is 7. ISO calendar algorithm taken from http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm """ year = self._year week1monday = _isoweek1monday(year) today = _ymd2ord(self._year, self._month, self._day) # Internally, week and day have origin 0 week, day = divmod(today - week1monday, 7) if week < 0: year -= 1 week1monday = _isoweek1monday(year) week, day = divmod(today - week1monday, 7) elif week >= 52: if today >= _isoweek1monday(year+1): year += 1 week = 0 return year, week+1, day+1 # Pickle support. def _getstate(self): yhi, ylo = divmod(self._year, 256) return (_struct.pack('4B', yhi, ylo, self._month, self._day),) def __setstate(self, string): yhi, ylo, self._month, self._day = (ord(string[0]), ord(string[1]), ord(string[2]), ord(string[3])) self._year = yhi * 256 + ylo def __reduce__(self): return (self.__class__, self._getstate()) _date_class = date # so functions w/ args named "date" can get at the class date.min = date(1, 1, 1) date.max = date(9999, 12, 31) date.resolution = timedelta(days=1) class tzinfo(object): """Abstract base class for time zone info classes. Subclasses must override the name(), utcoffset() and dst() methods. """ __slots__ = () def tzname(self, dt): "datetime -> string name of time zone." raise NotImplementedError("tzinfo subclass must override tzname()") def utcoffset(self, dt): "datetime -> minutes east of UTC (negative for west of UTC)" raise NotImplementedError("tzinfo subclass must override utcoffset()") def dst(self, dt): """datetime -> DST offset in minutes east of UTC. Return 0 if DST not in effect. utcoffset() must include the DST offset. """ raise NotImplementedError("tzinfo subclass must override dst()") def fromutc(self, dt): "datetime in UTC -> datetime in local time." if not isinstance(dt, datetime): raise TypeError("fromutc() requires a datetime argument") if dt.tzinfo is not self: raise ValueError("dt.tzinfo is not self") dtoff = dt.utcoffset() if dtoff is None: raise ValueError("fromutc() requires a non-None utcoffset() " "result") # See the long comment block at the end of this file for an # explanation of this algorithm. dtdst = dt.dst() if dtdst is None: raise ValueError("fromutc() requires a non-None dst() result") delta = dtoff - dtdst if delta: dt += delta dtdst = dt.dst() if dtdst is None: raise ValueError("fromutc(): dt.dst gave inconsistent " "results; cannot convert") if dtdst: return dt + dtdst else: return dt # Pickle support. def __reduce__(self): getinitargs = getattr(self, "__getinitargs__", None) if getinitargs: args = getinitargs() else: args = () getstate = getattr(self, "__getstate__", None) if getstate: state = getstate() else: state = getattr(self, "__dict__", None) or None if state is None: return (self.__class__, args) else: return (self.__class__, args, state) _tzinfo_class = tzinfo class time(object): """Time with time zone. Constructors: __new__() Operators: __repr__, __str__ __cmp__, __hash__ Methods: strftime() isoformat() utcoffset() tzname() dst() Properties (readonly): hour, minute, second, microsecond, tzinfo """ __slots__ = '_hour', '_minute', '_second', '_microsecond', '_tzinfo', '_hashcode' def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): """Constructor. Arguments: hour, minute (required) second, microsecond (default to zero) tzinfo (default to None) """ # if isinstance(hour, bytes) and len(hour) == 6 and ord(hour[0]) < 24: # # Pickle support # self = object.__new__(cls) # self.__setstate(hour, minute or None) # self._hashcode = -1 # return self hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) self = object.__new__(cls) self._hour = hour self._minute = minute self._second = second self._microsecond = microsecond self._tzinfo = tzinfo self._hashcode = -1 return self # Read-only field accessors @property def hour(self): """hour (0-23)""" return self._hour @property def minute(self): """minute (0-59)""" return self._minute @property def second(self): """second (0-59)""" return self._second @property def microsecond(self): """microsecond (0-999999)""" return self._microsecond @property def tzinfo(self): """timezone info object""" return self._tzinfo # Standard conversions, __hash__ (and helpers) # Comparisons of time objects with other. def __eq__(self, other): if isinstance(other, time): return self._cmp(other) == 0 else: return False def __ne__(self, other): if isinstance(other, time): return self._cmp(other) != 0 else: return True def __le__(self, other): if isinstance(other, time): return self._cmp(other) <= 0 else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, time): return self._cmp(other) < 0 else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, time): return self._cmp(other) >= 0 else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, time): return self._cmp(other) > 0 else: _cmperror(self, other) def _cmp(self, other): assert isinstance(other, time) mytz = self._tzinfo ottz = other._tzinfo myoff = otoff = None if mytz is ottz: base_compare = True else: myoff = self._utcoffset() otoff = other._utcoffset() base_compare = myoff == otoff if base_compare: return _cmp((self._hour, self._minute, self._second, self._microsecond), (other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: raise TypeError("can't compare offset-naive and offset-aware times") myhhmm = self._hour * 60 + self._minute - myoff othhmm = other._hour * 60 + other._minute - otoff return _cmp((myhhmm, self._second, self._microsecond), (othhmm, other._second, other._microsecond)) def __hash__(self): """Hash.""" if self._hashcode == -1: tzoff = self._utcoffset() if not tzoff: # zero or None self._hashcode = hash(self._getstate()[0]) else: h, m = divmod(self.hour * 60 + self.minute - tzoff, 60) if 0 <= h < 24: self._hashcode = hash(time(h, m, self.second, self.microsecond)) else: self._hashcode = hash((h, m, self.second, self.microsecond)) return self._hashcode # Conversion to string def _tzstr(self, sep=":"): """Return formatted timezone offset (+xx:xx) or None.""" off = self._utcoffset() if off is not None: if off < 0: sign = "-" off = -off else: sign = "+" hh, mm = divmod(off, 60) assert 0 <= hh < 24 off = "%s%02d%s%02d" % (sign, hh, sep, mm) return off def __repr__(self): """Convert to formal string, for repr().""" if self._microsecond != 0: s = ", %d, %d" % (self._second, self._microsecond) elif self._second != 0: s = ", %d" % self._second else: s = "" module = "datetime." if self.__class__ is time else "" s= "%s(%d, %d%s)" % (module + self.__class__.__name__, self._hour, self._minute, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" return s def isoformat(self): """Return the time formatted according to ISO. This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if self.microsecond == 0. """ s = _format_time(self._hour, self._minute, self._second, self._microsecond) tz = self._tzstr() if tz: s += tz return s __str__ = isoformat # def strftime(self, format): # """Format using strftime(). The date part of the timestamp passed # to underlying strftime should not be used. # """ # # The year must be >= _MINYEARFMT else Python's strftime implementation # # can raise a bogus exception. # timetuple = (1900, 1, 1, # self._hour, self._minute, self._second, # 0, 1, -1) # return _wrap_strftime(self, format, timetuple) def __format__(self, fmt): if not isinstance(fmt, (str, unicode)): raise ValueError("__format__ expects str or unicode, not %s" % fmt.__class__.__name__) if len(fmt) != 0: return self.strftime(fmt) return str(self) # Timezone functions def utcoffset(self): """Return the timezone offset in minutes east of UTC (negative west of UTC).""" if self._tzinfo is None: return None offset = self._tzinfo.utcoffset(None) offset = _check_utc_offset("utcoffset", offset) if offset is not None: offset = timedelta._create(0, offset * 60, 0, True) return offset # Return an integer (or None) instead of a timedelta (or None). def _utcoffset(self): if self._tzinfo is None: return None offset = self._tzinfo.utcoffset(None) offset = _check_utc_offset("utcoffset", offset) return offset def tzname(self): """Return the timezone name. Note that the name is 100% informational -- there's no requirement that it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. """ if self._tzinfo is None: return None name = self._tzinfo.tzname(None) _check_tzname(name) return name def dst(self): """Return 0 if DST is not in effect, or the DST offset (in minutes eastward) if DST is in effect. This is purely informational; the DST offset has already been added to the UTC offset returned by utcoffset() if applicable, so there's no need to consult dst() unless you're interested in displaying the DST info. """ if self._tzinfo is None: return None offset = self._tzinfo.dst(None) offset = _check_utc_offset("dst", offset) if offset is not None: offset = timedelta._create(0, offset * 60, 0, True) return offset # Return an integer (or None) instead of a timedelta (or None). def _dst(self): if self._tzinfo is None: return None offset = self._tzinfo.dst(None) offset = _check_utc_offset("dst", offset) return offset def replace(self, hour=None, minute=None, second=None, microsecond=None, tzinfo=True): """Return a new time with new values for the specified fields.""" if hour is None: hour = self.hour if minute is None: minute = self.minute if second is None: second = self.second if microsecond is None: microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo return time.__new__(type(self), hour, minute, second, microsecond, tzinfo) def __nonzero__(self): if self.second or self.microsecond: return True offset = self._utcoffset() or 0 return self.hour * 60 + self.minute != offset # Pickle support. def _getstate(self): us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) basestate = _struct.pack('6B', self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) else: return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): raise TypeError("bad tzinfo state arg") self._hour, self._minute, self._second, us1, us2, us3 = ( ord(string[0]), ord(string[1]), ord(string[2]), ord(string[3]), ord(string[4]), ord(string[5])) self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): return (time, self._getstate()) _time_class = time # so functions w/ args named "time" can get at the class time.min = time(0, 0, 0) time.max = time(23, 59, 59, 999999) time.resolution = timedelta(microseconds=1) class datetime(date): """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints or longs. """ __slots__ = date.__slots__ + time.__slots__ def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): # if isinstance(year, bytes) and len(year) == 10 and \ # 1 <= ord(year[2]) <= 12: # # Pickle support # self = object.__new__(cls) # self.__setstate(year, month) # self._hashcode = -1 # return self year, month, day = _check_date_fields(year, month, day) hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) self = object.__new__(cls) self._year = year self._month = month self._day = day self._hour = hour self._minute = minute self._second = second self._microsecond = microsecond self._tzinfo = tzinfo self._hashcode = -1 return self # Read-only field accessors @property def hour(self): """hour (0-23)""" return self._hour @property def minute(self): """minute (0-59)""" return self._minute @property def second(self): """second (0-59)""" return self._second @property def microsecond(self): """microsecond (0-999999)""" return self._microsecond @property def tzinfo(self): """timezone info object""" return self._tzinfo @classmethod def fromtimestamp(cls, timestamp, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). A timezone info object may be passed in as well. """ _check_tzinfo_arg(tz) converter = _time.localtime if tz is None else _time.gmtime self = cls._from_timestamp(converter, timestamp, tz) if tz is not None: self = tz.fromutc(self) return self @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." return cls._from_timestamp(_time.gmtime, t, None) @classmethod def _from_timestamp(cls, converter, timestamp, tzinfo): t_full = timestamp timestamp = int(_math.floor(timestamp)) frac = t_full - timestamp us = _round(frac * 1e6) # If timestamp is less than one microsecond smaller than a # full second, us can be rounded up to 1000000. In this case, # roll over to seconds, otherwise, ValueError is raised # by the constructor. if us == 1000000: timestamp += 1 us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them return cls(y, m, d, hh, mm, ss, us, tzinfo) @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." t = _time.time() return cls.fromtimestamp(t, tz) @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." t = _time.time() return cls.utcfromtimestamp(t) @classmethod def combine(cls, date, time): "Construct a datetime from a given date and a given time." if not isinstance(date, _date_class): raise TypeError("date argument must be a date instance") if not isinstance(time, _time_class): raise TypeError("time argument must be a time instance") return cls(date.year, date.month, date.day, time.hour, time.minute, time.second, time.microsecond, time.tzinfo) def timetuple(self): "Return local time tuple compatible with time.localtime()." dst = self._dst() if dst is None: dst = -1 elif dst: dst = 1 return _build_struct_time(self.year, self.month, self.day, self.hour, self.minute, self.second, dst) def utctimetuple(self): "Return UTC time tuple compatible with time.gmtime()." y, m, d = self.year, self.month, self.day hh, mm, ss = self.hour, self.minute, self.second offset = self._utcoffset() if offset: # neither None nor 0 mm -= offset y, m, d, hh, mm, ss, _ = _normalize_datetime( y, m, d, hh, mm, ss, 0, ignore_overflow=True) return _build_struct_time(y, m, d, hh, mm, ss, 0) def date(self): "Return the date part." return date(self._year, self._month, self._day) def time(self): "Return the time part, with tzinfo None." return time(self.hour, self.minute, self.second, self.microsecond) def timetz(self): "Return the time part, with same tzinfo." return time(self.hour, self.minute, self.second, self.microsecond, self._tzinfo) def replace(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None, tzinfo=True): """Return a new datetime with new values for the specified fields.""" if year is None: year = self.year if month is None: month = self.month if day is None: day = self.day if hour is None: hour = self.hour if minute is None: minute = self.minute if second is None: second = self.second if microsecond is None: microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo return datetime.__new__(type(self), year, month, day, hour, minute, second, microsecond, tzinfo) def astimezone(self, tz): if not isinstance(tz, tzinfo): raise TypeError("tz argument must be an instance of tzinfo") mytz = self.tzinfo if mytz is None: raise ValueError("astimezone() requires an aware datetime") if tz is mytz: return self # Convert self to UTC, and attach the new time zone object. myoffset = self.utcoffset() if myoffset is None: raise ValueError("astimezone() requires an aware datetime") utc = (self - myoffset).replace(tzinfo=tz) # Convert from UTC to tz's local time. return tz.fromutc(utc) # Ways to produce a string. def ctime(self): "Return ctime() style string." weekday = self.toordinal() % 7 or 7 return "%s %s %2d %02d:%02d:%02d %04d" % ( _DAYNAMES[weekday], _MONTHNAMES[self._month], self._day, self._hour, self._minute, self._second, self._year) def isoformat(self, sep='T'): """Return the time formatted according to ISO. This is 'YYYY-MM-DD HH:MM:SS.mmmmmm', or 'YYYY-MM-DD HH:MM:SS' if self.microsecond == 0. If self.tzinfo is not None, the UTC offset is also attached, giving 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM' or 'YYYY-MM-DD HH:MM:SS+HH:MM'. Optional argument sep specifies the separator between date and time, default 'T'. """ s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + _format_time(self._hour, self._minute, self._second, self._microsecond)) off = self._utcoffset() if off is not None: if off < 0: sign = "-" off = -off else: sign = "+" hh, mm = divmod(off, 60) s += "%s%02d:%02d" % (sign, hh, mm) return s def __repr__(self): """Convert to formal string, for repr().""" L = [self._year, self._month, self._day, # These are never zero self._hour, self._minute, self._second, self._microsecond] if L[-1] == 0: del L[-1] if L[-1] == 0: del L[-1] s = ", ".join(map(str, L)) module = "datetime." if self.__class__ is datetime else "" s = "%s(%s)" % (module + self.__class__.__name__, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" return s def __str__(self): "Convert to string, for str()." return self.isoformat(sep=' ') # @classmethod # def strptime(cls, date_string, format): # 'string, format -> new datetime parsed from a string (like time.strptime()).' # from _strptime import _strptime # # _strptime._strptime returns a two-element tuple. The first # # element is a time.struct_time object. The second is the # # microseconds (which are not defined for time.struct_time). # struct, micros = _strptime(date_string, format) # return cls(*(struct[0:6] + (micros,))) def utcoffset(self): """Return the timezone offset in minutes east of UTC (negative west of UTC).""" if self._tzinfo is None: return None offset = self._tzinfo.utcoffset(self) offset = _check_utc_offset("utcoffset", offset) if offset is not None: offset = timedelta._create(0, offset * 60, 0, True) return offset # Return an integer (or None) instead of a timedelta (or None). def _utcoffset(self): if self._tzinfo is None: return None offset = self._tzinfo.utcoffset(self) offset = _check_utc_offset("utcoffset", offset) return offset def tzname(self): """Return the timezone name. Note that the name is 100% informational -- there's no requirement that it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. """ if self._tzinfo is None: return None name = self._tzinfo.tzname(self) _check_tzname(name) return name def dst(self): """Return 0 if DST is not in effect, or the DST offset (in minutes eastward) if DST is in effect. This is purely informational; the DST offset has already been added to the UTC offset returned by utcoffset() if applicable, so there's no need to consult dst() unless you're interested in displaying the DST info. """ if self._tzinfo is None: return None offset = self._tzinfo.dst(self) offset = _check_utc_offset("dst", offset) if offset is not None: offset = timedelta._create(0, offset * 60, 0, True) return offset # Return an integer (or None) instead of a timedelta (or None). def _dst(self): if self._tzinfo is None: return None offset = self._tzinfo.dst(self) offset = _check_utc_offset("dst", offset) return offset # Comparisons of datetime objects with other. def __eq__(self, other): if isinstance(other, datetime): return self._cmp(other) == 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: return False def __ne__(self, other): if isinstance(other, datetime): return self._cmp(other) != 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: return True def __le__(self, other): if isinstance(other, datetime): return self._cmp(other) <= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) def __lt__(self, other): if isinstance(other, datetime): return self._cmp(other) < 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) def __ge__(self, other): if isinstance(other, datetime): return self._cmp(other) >= 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) def __gt__(self, other): if isinstance(other, datetime): return self._cmp(other) > 0 elif hasattr(other, "timetuple") and not isinstance(other, date): return NotImplemented else: _cmperror(self, other) def _cmp(self, other): assert isinstance(other, datetime) mytz = self._tzinfo ottz = other._tzinfo myoff = otoff = None if mytz is ottz: base_compare = True else: if mytz is not None: myoff = self._utcoffset() if ottz is not None: otoff = other._utcoffset() base_compare = myoff == otoff if base_compare: return _cmp((self._year, self._month, self._day, self._hour, self._minute, self._second, self._microsecond), (other._year, other._month, other._day, other._hour, other._minute, other._second, other._microsecond)) if myoff is None or otoff is None: raise TypeError("can't compare offset-naive and offset-aware datetimes") # XXX What follows could be done more efficiently... diff = self - other # this will take offsets into account if diff.days < 0: return -1 return diff and 1 or 0 def _add_timedelta(self, other, factor): y, m, d, hh, mm, ss, us = _normalize_datetime( self._year, self._month, self._day + other.days * factor, self._hour, self._minute, self._second + other.seconds * factor, self._microsecond + other.microseconds * factor) return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) def __add__(self, other): "Add a datetime and a timedelta." if not isinstance(other, timedelta): return NotImplemented return self._add_timedelta(other, 1) __radd__ = __add__ def __sub__(self, other): "Subtract two datetimes, or a datetime and a timedelta." if not isinstance(other, datetime): if isinstance(other, timedelta): return self._add_timedelta(other, -1) return NotImplemented delta_d = self.toordinal() - other.toordinal() delta_s = (self._hour - other._hour) * 3600 + \ (self._minute - other._minute) * 60 + \ (self._second - other._second) delta_us = self._microsecond - other._microsecond base = timedelta._create(delta_d, delta_s, delta_us, True) if self._tzinfo is other._tzinfo: return base myoff = self._utcoffset() otoff = other._utcoffset() if myoff == otoff: return base if myoff is None or otoff is None: raise TypeError("can't subtract offset-naive and offset-aware datetimes") return base + timedelta(minutes = otoff-myoff) def __hash__(self): if self._hashcode == -1: tzoff = self._utcoffset() if tzoff is None: self._hashcode = hash(self._getstate()[0]) else: days = _ymd2ord(self.year, self.month, self.day) seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second self._hashcode = hash(timedelta(days, seconds, self.microsecond)) return self._hashcode # Pickle support. def _getstate(self): yhi, ylo = divmod(self._year, 256) us2, us3 = divmod(self._microsecond, 256) us1, us2 = divmod(us2, 256) basestate = _struct.pack('10B', yhi, ylo, self._month, self._day, self._hour, self._minute, self._second, us1, us2, us3) if self._tzinfo is None: return (basestate,) else: return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): raise TypeError("bad tzinfo state arg") (yhi, ylo, self._month, self._day, self._hour, self._minute, self._second, us1, us2, us3) = (ord(string[0]), ord(string[1]), ord(string[2]), ord(string[3]), ord(string[4]), ord(string[5]), ord(string[6]), ord(string[7]), ord(string[8]), ord(string[9])) self._year = yhi * 256 + ylo self._microsecond = (((us1 << 8) | us2) << 8) | us3 self._tzinfo = tzinfo def __reduce__(self): return (self.__class__, self._getstate()) datetime.min = datetime(1, 1, 1) datetime.max = datetime(9999, 12, 31, 23, 59, 59, 999999) datetime.resolution = timedelta(microseconds=1) def _isoweek1monday(year): # Helper to calculate the day number of the Monday starting week 1 # XXX This could be done more efficiently THURSDAY = 3 firstday = _ymd2ord(year, 1, 1) firstweekday = (firstday + 6) % 7 # See weekday() above week1monday = firstday - firstweekday if firstweekday > THURSDAY: week1monday += 7 return week1monday """ Some time zone algebra. For a datetime x, let x.n = x stripped of its timezone -- its naive time. x.o = x.utcoffset(), and assuming that doesn't raise an exception or return None x.d = x.dst(), and assuming that doesn't raise an exception or return None x.s = x's standard offset, x.o - x.d Now some derived rules, where k is a duration (timedelta). 1. x.o = x.s + x.d This follows from the definition of x.s. 2. If x and y have the same tzinfo member, x.s = y.s. This is actually a requirement, an assumption we need to make about sane tzinfo classes. 3. The naive UTC time corresponding to x is x.n - x.o. This is again a requirement for a sane tzinfo class. 4. (x+k).s = x.s This follows from #2, and that datimetimetz+timedelta preserves tzinfo. 5. (x+k).n = x.n + k Again follows from how arithmetic is defined. Now we can explain tz.fromutc(x). Let's assume it's an interesting case (meaning that the various tzinfo methods exist, and don't blow up or return None when called). The function wants to return a datetime y with timezone tz, equivalent to x. x is already in UTC. By #3, we want y.n - y.o = x.n [1] The algorithm starts by attaching tz to x.n, and calling that y. So x.n = y.n at the start. Then it wants to add a duration k to y, so that [1] becomes true; in effect, we want to solve [2] for k: (y+k).n - (y+k).o = x.n [2] By #1, this is the same as (y+k).n - ((y+k).s + (y+k).d) = x.n [3] By #5, (y+k).n = y.n + k, which equals x.n + k because x.n=y.n at the start. Substituting that into [3], x.n + k - (y+k).s - (y+k).d = x.n; the x.n terms cancel, leaving k - (y+k).s - (y+k).d = 0; rearranging, k = (y+k).s - (y+k).d; by #4, (y+k).s == y.s, so k = y.s - (y+k).d On the RHS, (y+k).d can't be computed directly, but y.s can be, and we approximate k by ignoring the (y+k).d term at first. Note that k can't be very large, since all offset-returning methods return a duration of magnitude less than 24 hours. For that reason, if y is firmly in std time, (y+k).d must be 0, so ignoring it has no consequence then. In any case, the new value is z = y + y.s [4] It's helpful to step back at look at [4] from a higher level: it's simply mapping from UTC to tz's standard time. At this point, if z.n - z.o = x.n [5] we have an equivalent time, and are almost done. The insecurity here is at the start of daylight time. Picture US Eastern for concreteness. The wall time jumps from 1:59 to 3:00, and wall hours of the form 2:MM don't make good sense then. The docs ask that an Eastern tzinfo class consider such a time to be EDT (because it's "after 2"), which is a redundant spelling of 1:MM EST on the day DST starts. We want to return the 1:MM EST spelling because that's the only spelling that makes sense on the local wall clock. In fact, if [5] holds at this point, we do have the standard-time spelling, but that takes a bit of proof. We first prove a stronger result. What's the difference between the LHS and RHS of [5]? Let diff = x.n - (z.n - z.o) [6] Now z.n = by [4] (y + y.s).n = by #5 y.n + y.s = since y.n = x.n x.n + y.s = since z and y are have the same tzinfo member, y.s = z.s by #2 x.n + z.s Plugging that back into [6] gives diff = x.n - ((x.n + z.s) - z.o) = expanding x.n - x.n - z.s + z.o = cancelling - z.s + z.o = by #2 z.d So diff = z.d. If [5] is true now, diff = 0, so z.d = 0 too, and we have the standard-time spelling we wanted in the endcase described above. We're done. Contrarily, if z.d = 0, then we have a UTC equivalent, and are also done. If [5] is not true now, diff = z.d != 0, and z.d is the offset we need to add to z (in effect, z is in tz's standard time, and we need to shift the local clock into tz's daylight time). Let z' = z + z.d = z + diff [7] and we can again ask whether z'.n - z'.o = x.n [8] If so, we're done. If not, the tzinfo class is insane, according to the assumptions we've made. This also requires a bit of proof. As before, let's compute the difference between the LHS and RHS of [8] (and skipping some of the justifications for the kinds of substitutions we've done several times already): diff' = x.n - (z'.n - z'.o) = replacing z'.n via [7] x.n - (z.n + diff - z'.o) = replacing diff via [6] x.n - (z.n + x.n - (z.n - z.o) - z'.o) = x.n - z.n - x.n + z.n - z.o + z'.o = cancel x.n - z.n + z.n - z.o + z'.o = cancel z.n - z.o + z'.o = #1 twice -z.s - z.d + z'.s + z'.d = z and z' have same tzinfo z'.d - z.d So z' is UTC-equivalent to x iff z'.d = z.d at this point. If they are equal, we've found the UTC-equivalent so are done. In fact, we stop with [7] and return z', not bothering to compute z'.d. How could z.d and z'd differ? z' = z + z.d [7], so merely moving z' by a dst() offset, and starting *from* a time already in DST (we know z.d != 0), would have to change the result dst() returns: we start in DST, and moving a little further into it takes us out of DST. There isn't a sane case where this can happen. The closest it gets is at the end of DST, where there's an hour in UTC with no spelling in a hybrid tzinfo class. In US Eastern, that's 5:MM UTC = 0:MM EST = 1:MM EDT. During that hour, on an Eastern clock 1:MM is taken as being in standard time (6:MM UTC) because the docs insist on that, but 0:MM is taken as being in daylight time (4:MM UTC). There is no local time mapping to 5:MM UTC. The local clock jumps from 1:59 back to 1:00 again, and repeats the 1:MM hour in standard time. Since that's what the local clock *does*, we want to map both UTC hours 5:MM and 6:MM to 1:MM Eastern. The result is ambiguous in local time, but so it goes -- it's the way the local clock works. When x = 5:MM UTC is the input to this algorithm, x.o=0, y.o=-5 and y.d=0, so z=0:MM. z.d=60 (minutes) then, so [5] doesn't hold and we keep going. z' = z + z.d = 1:MM then, and z'.d=0, and z'.d - z.d = -60 != 0 so [8] (correctly) concludes that z' is not UTC-equivalent to x. Because we know z.d said z was in daylight time (else [5] would have held and we would have stopped then), and we know z.d != z'.d (else [8] would have held and we have stopped then), and there are only 2 possible values dst() can return in Eastern, it follows that z'.d must be 0 (which it is in the example, but the reasoning doesn't depend on the example -- it depends on there being two possible dst() outcomes, one zero and the other non-zero). Therefore z' must be in standard time, and is the spelling we want in this case. Note again that z' is not UTC-equivalent as far as the hybrid tzinfo class is concerned (because it takes z' as being in standard time rather than the daylight time we intend here), but returning it gives the real-life "local clock repeats an hour" behavior when mapping the "unspellable" UTC hour into tz. When the input is 6:MM, z=1:MM and z.d=0, and we stop at once, again with the 1:MM standard time spelling we want. So how can this break? One of the assumptions must be violated. Two possibilities: 1) [2] effectively says that y.s is invariant across all y belong to a given time zone. This isn't true if, for political reasons or continental drift, a region decides to change its base offset from UTC. 2) There may be versions of "double daylight" time where the tail end of the analysis gives up a step too early. I haven't thought about that enough to say. In any case, it's clear that the default fromutc() is strong enough to handle "almost all" time zones: so long as the standard offset is invariant, it doesn't matter if daylight time transition points change from year to year, or if daylight time is skipped in some years; it doesn't matter how large or small dst() may get within its bounds; and it doesn't even matter if some perverse time zone returns a negative dst()). So a breaking case must be pretty bizarre, and a tzinfo subclass can override fromutc() if it is. """ ================================================ FILE: third_party/pythonparser/LICENSE.txt ================================================ Copyright (c) 2015 whitequark MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: third_party/pythonparser/README.md ================================================ The source code in this directory is forked from [github.com/m-labs/pythonparser](https://github.com/m-labs/pythonparser). There are very light modifications to the source code so that it will work with Grumpy. ================================================ FILE: third_party/pythonparser/__init__.py ================================================ from __future__ import absolute_import, division, print_function, unicode_literals import sys from . import source as pythonparser_source, lexer as pythonparser_lexer, parser as pythonparser_parser, diagnostic as pythonparser_diagnostic def parse_buffer(buffer, mode="exec", flags=[], version=None, engine=None): """ Like :meth:`parse`, but accepts a :class:`source.Buffer` instead of source and filename, and returns comments as well. :see: :meth:`parse` :return: (:class:`ast.AST`, list of :class:`source.Comment`) Abstract syntax tree and comments """ if version is None: version = sys.version_info[0:2] if engine is None: engine = pythonparser_diagnostic.Engine() lexer = pythonparser_lexer.Lexer(buffer, version, engine) if mode in ("single", "eval"): lexer.interactive = True parser = pythonparser_parser.Parser(lexer, version, engine) parser.add_flags(flags) if mode == "exec": return parser.file_input(), lexer.comments elif mode == "single": return parser.single_input(), lexer.comments elif mode == "eval": return parser.eval_input(), lexer.comments def parse(source, filename="", mode="exec", flags=[], version=None, engine=None): """ Parse a string into an abstract syntax tree. This is the replacement for the built-in :meth:`..ast.parse`. :param source: (string) Source code in the correct encoding :param filename: (string) Filename of the source (used in diagnostics) :param mode: (string) Execution mode. Pass ``"exec"`` to parse a module, ``"single"`` to parse a single (interactive) statement, and ``"eval"`` to parse an expression. In the last two cases, ``source`` must be terminated with an empty line (i.e. end with ``"\\n\\n"``). :param flags: (list of string) Future flags. Equivalent to ``from __future__ import ``. :param version: (2-tuple of int) Major and minor version of Python syntax to recognize, ``sys.version_info[0:2]`` by default. :param engine: (:class:`diagnostic.Engine`) Diagnostic engine, a fresh one is created by default :return: (:class:`ast.AST`) Abstract syntax tree :raise: :class:`diagnostic.Error` if the source code is not well-formed """ ast, comments = parse_buffer(pythonparser_source.Buffer(source, filename), mode, flags, version, engine) return ast ================================================ FILE: third_party/pythonparser/algorithm.py ================================================ """ The :mod:`Diagnostic` module provides several commonly useful algorithms that operate on abstract syntax trees. """ from __future__ import absolute_import, division, print_function, unicode_literals from . import ast class Visitor: """ A node visitor base class that does a traversal of the abstract syntax tree. This class is meant to be subclassed, with the subclass adding visitor methods. The visitor method should call ``self.generic_visit(node)`` to continue the traversal; this allows to perform arbitrary actions both before and after traversing the children of a node. The visitor methods for the nodes are ``'visit_'`` + class name of the node. So a `Try` node visit function would be `visit_Try`. """ def generic_visit(self, node): """Called if no explicit visitor function exists for a node.""" for field_name in node._fields: self.visit(getattr(node, field_name)) def _visit_one(self, node): visit_attr = "visit_" + type(node).__name__ if hasattr(self, visit_attr): return getattr(self, visit_attr)(node) else: return self.generic_visit(node) def visit(self, obj): """Visit a node or a list of nodes. Other values are ignored""" if isinstance(obj, list): return [self.visit(elt) for elt in obj] elif isinstance(obj, ast.AST): return self._visit_one(obj) class Transformer: """ A node transformer base class that does a post-order traversal of the abstract syntax tree while allowing to replace or remove the nodes being traversed. The return value of the visitor methods is used to replace or remove the old node. If the return value of the visitor method is ``None``, the node will be removed from its location, otherwise it is replaced with the return value. The return value may be the original node in which case no replacement takes place. This class is meant to be subclassed, with the subclass adding visitor methods. The visitor method should call ``self.generic_visit(node)`` to continue the traversal; this allows to perform arbitrary actions both before and after traversing the children of a node. The visitor methods for the nodes are ``'visit_'`` + class name of the node. So a `Try` node visit function would be `visit_Try`. """ def generic_visit(self, node): """Called if no explicit visitor function exists for a node.""" for field_name in node._fields: setattr(node, field_name, self.visit(getattr(node, field_name))) return node def _visit_one(self, node): visit_attr = "visit_" + type(node).__name__ if hasattr(self, visit_attr): return getattr(self, visit_attr)(node) else: return self.generic_visit(node) def visit(self, obj): """Visit a node or a list of nodes. Other values are ignored""" if isinstance(obj, list): return list(filter(lambda x: x is not None, map(self.visit, obj))) elif isinstance(obj, ast.AST): return self._visit_one(obj) else: return obj def compare(left, right, compare_locs=False): """ An AST comparison function. Returns ``True`` if all fields in ``left`` are equal to fields in ``right``; if ``compare_locs`` is true, all locations should match as well. """ if type(left) != type(right): return False if isinstance(left, ast.AST): for field in left._fields: if not compare(getattr(left, field), getattr(right, field)): return False if compare_locs: for loc in left._locs: if getattr(left, loc) != getattr(right, loc): return False return True elif isinstance(left, list): if len(left) != len(right): return False for left_elt, right_elt in zip(left, right): if not compare(left_elt, right_elt): return False return True else: return left == right ================================================ FILE: third_party/pythonparser/ast.py ================================================ # encoding: utf-8 """ The :mod:`ast` module contains the classes comprising the Python abstract syntax tree. All attributes ending with ``loc`` contain instances of :class:`.source.Range` or None. All attributes ending with ``_locs`` contain lists of instances of :class:`.source.Range` or []. The attribute ``loc``, present in every class except those inheriting :class:`boolop`, has a special meaning: it encompasses the entire AST node, so that it is possible to cut the range contained inside ``loc`` of a parsetree fragment and paste it somewhere else without altering said parsetree fragment that. The AST format for all supported versions is generally normalized to be a superset of the native :mod:`..ast` module of the latest supported Python version. In particular this affects: * :class:`With`: on 2.6-2.7 it uses the 3.0 format. * :class:`TryExcept` and :class:`TryFinally`: on 2.6-2.7 they're replaced with :class:`Try` from 3.0. * :class:`arguments`: on 2.6-3.1 it uses the 3.2 format, with dedicated :class:`arg` in ``vararg`` and ``kwarg`` slots. """ from __future__ import absolute_import, division, print_function, unicode_literals # Location mixins class commonloc(object): """ A mixin common for all nodes. :cvar _locs: (tuple of strings) names of all attributes with location values :ivar loc: range encompassing all locations defined for this node or its children """ _locs = ("loc",) def _reprfields(self): return self._fields + self._locs def __repr__(self): def value(name): try: loc = self.__dict__[name] if isinstance(loc, list): return "[%s]" % (", ".join(map(repr, loc))) else: return repr(loc) except: return "(!!!MISSING!!!)" fields = ", ".join(map(lambda name: "%s=%s" % (name, value(name)), self._reprfields())) return "%s(%s)" % (self.__class__.__name__, fields) @property def lineno(self): return self.loc.line() class keywordloc(commonloc): """ A mixin common for all keyword statements, e.g. ``pass`` and ``yield expr``. :ivar keyword_loc: location of the keyword, e.g. ``yield``. """ _locs = commonloc._locs + ("keyword_loc",) class beginendloc(commonloc): """ A mixin common for nodes with a opening and closing delimiters, e.g. tuples and lists. :ivar begin_loc: location of the opening delimiter, e.g. ``(``. :ivar end_loc: location of the closing delimiter, e.g. ``)``. """ _locs = commonloc._locs + ("begin_loc", "end_loc") # AST nodes class AST(object): """ An ancestor of all nodes. :cvar _fields: (tuple of strings) names of all attributes with semantic values """ _fields = () def __init__(self, **fields): for field in fields: setattr(self, field, fields[field]) class alias(AST, commonloc): """ An import alias, e.g. ``x as y``. :ivar name: (string) value to import :ivar asname: (string) name to add to the environment :ivar name_loc: location of name :ivar as_loc: location of ``as`` :ivar asname_loc: location of asname """ _fields = ("name", "asname") _locs = commonloc._locs + ("name_loc", "as_loc", "asname_loc") class arg(AST, commonloc): """ A formal argument, e.g. in ``def f(x)`` or ``def f(x: T)``. :ivar arg: (string) argument name :ivar annotation: (:class:`AST`) type annotation, if any; **emitted since 3.0** :ivar arg_loc: location of argument name :ivar colon_loc: location of ``:``, if any; **emitted since 3.0** """ _fields = ("arg", "annotation") _locs = commonloc._locs + ("arg_loc", "colon_loc") class arguments(AST, beginendloc): """ Function definition arguments, e.g. in ``def f(x, y=1, *z, **t)``. :ivar args: (list of :class:`arg`) regular formal arguments :ivar defaults: (list of :class:`AST`) values of default arguments :ivar vararg: (:class:`arg`) splat formal argument (if any), e.g. in ``*x`` :ivar kwonlyargs: (list of :class:`arg`) keyword-only (post-\*) formal arguments; **emitted since 3.0** :ivar kw_defaults: (list of :class:`AST`) values of default keyword-only arguments; **emitted since 3.0** :ivar kwarg: (:class:`arg`) keyword splat formal argument (if any), e.g. in ``**x`` :ivar star_loc: location of ``*``, if any :ivar dstar_loc: location of ``**``, if any :ivar equals_locs: locations of ``=`` :ivar kw_equals_locs: locations of ``=`` of default keyword-only arguments; **emitted since 3.0** """ _fields = ("args", "vararg", "kwonlyargs", "kwarg", "defaults", "kw_defaults") _locs = beginendloc._locs + ("star_loc", "dstar_loc", "equals_locs", "kw_equals_locs") class boolop(AST, commonloc): """ Base class for binary boolean operators. This class is unlike others in that it does not have the ``loc`` field. It serves only as an indicator of operation and corresponds to no source itself; locations are recorded in :class:`BoolOp`. """ _locs = () class And(boolop): """The ``and`` operator.""" class Or(boolop): """The ``or`` operator.""" class cmpop(AST, commonloc): """Base class for comparison operators.""" class Eq(cmpop): """The ``==`` operator.""" class Gt(cmpop): """The ``>`` operator.""" class GtE(cmpop): """The ``>=`` operator.""" class In(cmpop): """The ``in`` operator.""" class Is(cmpop): """The ``is`` operator.""" class IsNot(cmpop): """The ``is not`` operator.""" class Lt(cmpop): """The ``<`` operator.""" class LtE(cmpop): """The ``<=`` operator.""" class NotEq(cmpop): """The ``!=`` (or deprecated ``<>``) operator.""" class NotIn(cmpop): """The ``not in`` operator.""" class comprehension(AST, commonloc): """ A single ``for`` list comprehension clause. :ivar target: (assignable :class:`AST`) the variable(s) bound in comprehension body :ivar iter: (:class:`AST`) the expression being iterated :ivar ifs: (list of :class:`AST`) the ``if`` clauses :ivar for_loc: location of the ``for`` keyword :ivar in_loc: location of the ``in`` keyword :ivar if_locs: locations of ``if`` keywords """ _fields = ("target", "iter", "ifs") _locs = commonloc._locs + ("for_loc", "in_loc", "if_locs") class excepthandler(AST, commonloc): """Base class for the exception handler.""" class ExceptHandler(excepthandler): """ An exception handler, e.g. ``except x as y:· z``. :ivar type: (:class:`AST`) type of handled exception, if any :ivar name: (assignable :class:`AST` **until 3.0**, string **since 3.0**) variable bound to exception, if any :ivar body: (list of :class:`AST`) code to execute when exception is caught :ivar except_loc: location of ``except`` :ivar as_loc: location of ``as``, if any :ivar name_loc: location of variable name :ivar colon_loc: location of ``:`` """ _fields = ("type", "name", "body") _locs = excepthandler._locs + ("except_loc", "as_loc", "name_loc", "colon_loc") class expr(AST, commonloc): """Base class for expression nodes.""" class Attribute(expr): """ An attribute access, e.g. ``x.y``. :ivar value: (:class:`AST`) left-hand side :ivar attr: (string) attribute name """ _fields = ("value", "attr", "ctx") _locs = expr._locs + ("dot_loc", "attr_loc") class BinOp(expr): """ A binary operation, e.g. ``x + y``. :ivar left: (:class:`AST`) left-hand side :ivar op: (:class:`operator`) operator :ivar right: (:class:`AST`) right-hand side """ _fields = ("left", "op", "right") class BoolOp(expr): """ A boolean operation, e.g. ``x and y``. :ivar op: (:class:`boolop`) operator :ivar values: (list of :class:`AST`) operands :ivar op_locs: locations of operators """ _fields = ("op", "values") _locs = expr._locs + ("op_locs",) class Call(expr, beginendloc): """ A function call, e.g. ``f(x, y=1, *z, **t)``. :ivar func: (:class:`AST`) function to call :ivar args: (list of :class:`AST`) regular arguments :ivar keywords: (list of :class:`keyword`) keyword arguments :ivar starargs: (:class:`AST`) splat argument (if any), e.g. in ``*x`` :ivar kwargs: (:class:`AST`) keyword splat argument (if any), e.g. in ``**x`` :ivar star_loc: location of ``*``, if any :ivar dstar_loc: location of ``**``, if any """ _fields = ("func", "args", "keywords", "starargs", "kwargs") _locs = beginendloc._locs + ("star_loc", "dstar_loc") class Compare(expr): """ A comparison operation, e.g. ``x < y`` or ``x < y > z``. :ivar left: (:class:`AST`) left-hand :ivar ops: (list of :class:`cmpop`) compare operators :ivar comparators: (list of :class:`AST`) compare values """ _fields = ("left", "ops", "comparators") class Dict(expr, beginendloc): """ A dictionary, e.g. ``{x: y}``. :ivar keys: (list of :class:`AST`) keys :ivar values: (list of :class:`AST`) values :ivar colon_locs: locations of ``:`` """ _fields = ("keys", "values") _locs = beginendloc._locs + ("colon_locs",) class DictComp(expr, beginendloc): """ A list comprehension, e.g. ``{x: y for x,y in z}``. **Emitted since 2.7.** :ivar key: (:class:`AST`) key part of comprehension body :ivar value: (:class:`AST`) value part of comprehension body :ivar generators: (list of :class:`comprehension`) ``for`` clauses :ivar colon_loc: location of ``:`` """ _fields = ("key", "value", "generators") _locs = beginendloc._locs + ("colon_loc",) class Ellipsis(expr): """The ellipsis, e.g. in ``x[...]``.""" class GeneratorExp(expr, beginendloc): """ A generator expression, e.g. ``(x for x in y)``. :ivar elt: (:class:`AST`) expression body :ivar generators: (list of :class:`comprehension`) ``for`` clauses """ _fields = ("elt", "generators") class IfExp(expr): """ A conditional expression, e.g. ``x if y else z``. :ivar test: (:class:`AST`) condition :ivar body: (:class:`AST`) value if true :ivar orelse: (:class:`AST`) value if false :ivar if_loc: location of ``if`` :ivar else_loc: location of ``else`` """ _fields = ("test", "body", "orelse") _locs = expr._locs + ("if_loc", "else_loc") class Lambda(expr): """ A lambda expression, e.g. ``lambda x: x*x``. :ivar args: (:class:`arguments`) arguments :ivar body: (:class:`AST`) body :ivar lambda_loc: location of ``lambda`` :ivar colon_loc: location of ``:`` """ _fields = ("args", "body") _locs = expr._locs + ("lambda_loc", "colon_loc") class List(expr, beginendloc): """ A list, e.g. ``[x, y]``. :ivar elts: (list of :class:`AST`) elements """ _fields = ("elts", "ctx") class ListComp(expr, beginendloc): """ A list comprehension, e.g. ``[x for x in y]``. :ivar elt: (:class:`AST`) comprehension body :ivar generators: (list of :class:`comprehension`) ``for`` clauses """ _fields = ("elt", "generators") class Name(expr): """ An identifier, e.g. ``x``. :ivar id: (string) name """ _fields = ("id", "ctx") class NameConstant(expr): """ A named constant, e.g. ``None``. :ivar value: Python value, one of ``None``, ``True`` or ``False`` """ _fields = ("value",) class Num(expr): """ An integer, floating point or complex number, e.g. ``1``, ``1.0`` or ``1.0j``. :ivar n: (int, float or complex) value """ _fields = ("n",) class Repr(expr, beginendloc): """ A repr operation, e.g. ``\`x\``` **Emitted until 3.0.** :ivar value: (:class:`AST`) value """ _fields = ("value",) class Set(expr, beginendloc): """ A set, e.g. ``{x, y}``. **Emitted since 2.7.** :ivar elts: (list of :class:`AST`) elements """ _fields = ("elts",) class SetComp(expr, beginendloc): """ A set comprehension, e.g. ``{x for x in y}``. **Emitted since 2.7.** :ivar elt: (:class:`AST`) comprehension body :ivar generators: (list of :class:`comprehension`) ``for`` clauses """ _fields = ("elt", "generators") class Str(expr, beginendloc): """ A string, e.g. ``"x"``. :ivar s: (string) value """ _fields = ("s",) class Starred(expr): """ A starred expression, e.g. ``*x`` in ``*x, y = z``. :ivar value: (:class:`AST`) expression :ivar star_loc: location of ``*`` """ _fields = ("value", "ctx") _locs = expr._locs + ("star_loc",) class Subscript(expr, beginendloc): """ A subscript operation, e.g. ``x[1]``. :ivar value: (:class:`AST`) object being sliced :ivar slice: (:class:`slice`) slice """ _fields = ("value", "slice", "ctx") class Tuple(expr, beginendloc): """ A tuple, e.g. ``(x,)`` or ``x,y``. :ivar elts: (list of nodes) elements """ _fields = ("elts", "ctx") class UnaryOp(expr): """ An unary operation, e.g. ``+x``. :ivar op: (:class:`unaryop`) operator :ivar operand: (:class:`AST`) operand """ _fields = ("op", "operand") class Yield(expr): """ A yield expression, e.g. ``yield x``. :ivar value: (:class:`AST`) yielded value :ivar yield_loc: location of ``yield`` """ _fields = ("value",) _locs = expr._locs + ("yield_loc",) class YieldFrom(expr): """ A yield from expression, e.g. ``yield from x``. :ivar value: (:class:`AST`) yielded value :ivar yield_loc: location of ``yield`` :ivar from_loc: location of ``from`` """ _fields = ("value",) _locs = expr._locs + ("yield_loc", "from_loc") # expr_context # AugLoad # AugStore # Del # Load # Param # Store class keyword(AST, commonloc): """ A keyword actual argument, e.g. in ``f(x=1)``. :ivar arg: (string) name :ivar value: (:class:`AST`) value :ivar equals_loc: location of ``=`` """ _fields = ("arg", "value") _locs = commonloc._locs + ("arg_loc", "equals_loc") class mod(AST, commonloc): """Base class for modules (groups of statements).""" _fields = ("body",) class Expression(mod): """A group of statements parsed as if for :func:`eval`.""" class Interactive(mod): """A group of statements parsed as if it was REPL input.""" class Module(mod): """A group of statements parsed as if it was a file.""" class operator(AST, commonloc): """Base class for numeric binary operators.""" class Add(operator): """The ``+`` operator.""" class BitAnd(operator): """The ``&`` operator.""" class BitOr(operator): """The ``|`` operator.""" class BitXor(operator): """The ``^`` operator.""" class Div(operator): """The ``\\`` operator.""" class FloorDiv(operator): """The ``\\\\`` operator.""" class LShift(operator): """The ``<<`` operator.""" class MatMult(operator): """The ``@`` operator.""" class Mod(operator): """The ``%`` operator.""" class Mult(operator): """The ``*`` operator.""" class Pow(operator): """The ``**`` operator.""" class RShift(operator): """The ``>>`` operator.""" class Sub(operator): """The ``-`` operator.""" class slice(AST, commonloc): """Base class for slice operations.""" class ExtSlice(slice): """ The multiple slice, e.g. in ``x[0:1, 2:3]``. Note that multiple slices with only integer indexes will appear as instances of :class:`Index`. :ivar dims: (:class:`slice`) sub-slices """ _fields = ("dims",) class Index(slice): """ The index, e.g. in ``x[1]`` or ``x[1, 2]``. :ivar value: (:class:`AST`) index """ _fields = ("value",) class Slice(slice): """ The slice, e.g. in ``x[0:1]`` or ``x[0:1:2]``. :ivar lower: (:class:`AST`) lower bound, if any :ivar upper: (:class:`AST`) upper bound, if any :ivar step: (:class:`AST`) iteration step, if any :ivar bound_colon_loc: location of first semicolon :ivar step_colon_loc: location of second semicolon, if any """ _fields = ("lower", "upper", "step") _locs = slice._locs + ("bound_colon_loc", "step_colon_loc") class stmt(AST, commonloc): """Base class for statement nodes.""" class Assert(stmt, keywordloc): """ The ``assert x, msg`` statement. :ivar test: (:class:`AST`) condition :ivar msg: (:class:`AST`) message, if any """ _fields = ("test", "msg") class Assign(stmt): """ The ``=`` statement, e.g. in ``x = 1`` or ``x = y = 1``. :ivar targets: (list of assignable :class:`AST`) left-hand sides :ivar value: (:class:`AST`) right-hand side :ivar op_locs: location of equality signs corresponding to ``targets`` """ _fields = ("targets", "value") _locs = stmt._locs + ("op_locs",) class AugAssign(stmt): """ The operator-assignment statement, e.g. ``+=``. :ivar target: (assignable :class:`AST`) left-hand side :ivar op: (:class:`operator`) operator :ivar value: (:class:`AST`) right-hand side """ _fields = ("target", "op", "value") class Break(stmt, keywordloc): """The ``break`` statement.""" class ClassDef(stmt, keywordloc): """ The ``class x(z, y):· t`` (2.6) or ``class x(y, z=1, *t, **u):· v`` (3.0) statement. :ivar name: (string) name :ivar bases: (list of :class:`AST`) base classes :ivar keywords: (list of :class:`keyword`) keyword arguments; **emitted since 3.0** :ivar starargs: (:class:`AST`) splat argument (if any), e.g. in ``*x``; **emitted since 3.0** :ivar kwargs: (:class:`AST`) keyword splat argument (if any), e.g. in ``**x``; **emitted since 3.0** :ivar body: (list of :class:`AST`) body :ivar decorator_list: (list of :class:`AST`) decorators :ivar keyword_loc: location of ``class`` :ivar name_loc: location of name :ivar lparen_loc: location of ``(``, if any :ivar star_loc: location of ``*``, if any; **emitted since 3.0** :ivar dstar_loc: location of ``**``, if any; **emitted since 3.0** :ivar rparen_loc: location of ``)``, if any :ivar colon_loc: location of ``:`` :ivar at_locs: locations of decorator ``@`` """ _fields = ("name", "bases", "keywords", "starargs", "kwargs", "body", "decorator_list") _locs = keywordloc._locs + ("name_loc", "lparen_loc", "star_loc", "dstar_loc", "rparen_loc", "colon_loc", "at_locs") class Continue(stmt, keywordloc): """The ``continue`` statement.""" class Delete(stmt, keywordloc): """ The ``del x, y`` statement. :ivar targets: (list of :class:`Name`) """ _fields = ("targets",) class Exec(stmt, keywordloc): """ The ``exec code in locals, globals`` statement. **Emitted until 3.0.** :ivar body: (:class:`AST`) code :ivar locals: (:class:`AST`) locals :ivar globals: (:class:`AST`) globals :ivar keyword_loc: location of ``exec`` :ivar in_loc: location of ``in`` """ _fields = ("body", "locals", "globals") _locs = keywordloc._locs + ("in_loc",) class Expr(stmt): """ An expression in statement context. The value of expression is discarded. :ivar value: (:class:`expr`) value """ _fields = ("value",) class For(stmt, keywordloc): """ The ``for x in y:· z·else:· t`` statement. :ivar target: (assignable :class:`AST`) loop variable :ivar iter: (:class:`AST`) loop collection :ivar body: (list of :class:`AST`) code for every iteration :ivar orelse: (list of :class:`AST`) code if empty :ivar keyword_loc: location of ``for`` :ivar in_loc: location of ``in`` :ivar for_colon_loc: location of colon after ``for`` :ivar else_loc: location of ``else``, if any :ivar else_colon_loc: location of colon after ``else``, if any """ _fields = ("target", "iter", "body", "orelse") _locs = keywordloc._locs + ("in_loc", "for_colon_loc", "else_loc", "else_colon_loc") class FunctionDef(stmt, keywordloc): """ The ``def f(x):· y`` (2.6) or ``def f(x) -> t:· y`` (3.0) statement. :ivar name: (string) name :ivar args: (:class:`arguments`) formal arguments :ivar returns: (:class:`AST`) return type annotation; **emitted since 3.0** :ivar body: (list of :class:`AST`) body :ivar decorator_list: (list of :class:`AST`) decorators :ivar keyword_loc: location of ``def`` :ivar name_loc: location of name :ivar arrow_loc: location of ``->``, if any; **emitted since 3.0** :ivar colon_loc: location of ``:``, if any :ivar at_locs: locations of decorator ``@`` """ _fields = ("name", "args", "returns", "body", "decorator_list") _locs = keywordloc._locs + ("name_loc", "arrow_loc", "colon_loc", "at_locs") class Global(stmt, keywordloc): """ The ``global x, y`` statement. :ivar names: (list of string) names :ivar name_locs: locations of names """ _fields = ("names",) _locs = keywordloc._locs + ("name_locs",) class If(stmt, keywordloc): """ The ``if x:· y·else:· z`` or ``if x:· y·elif: z· t`` statement. :ivar test: (:class:`AST`) condition :ivar body: (list of :class:`AST`) code if true :ivar orelse: (list of :class:`AST`) code if false :ivar if_colon_loc: location of colon after ``if`` or ``elif`` :ivar else_loc: location of ``else``, if any :ivar else_colon_loc: location of colon after ``else``, if any """ _fields = ("test", "body", "orelse") _locs = keywordloc._locs + ("if_colon_loc", "else_loc", "else_colon_loc") class Import(stmt, keywordloc): """ The ``import x, y`` statement. :ivar names: (list of :class:`alias`) names """ _fields = ("names",) class ImportFrom(stmt, keywordloc): """ The ``from ...x import y, z`` or ``from x import (y, z)`` or ``from x import *`` statement. :ivar names: (list of :class:`alias`) names :ivar module: (string) module name, if any :ivar level: (integer) amount of dots before module name :ivar keyword_loc: location of ``from`` :ivar dots_loc: location of dots, if any :ivar module_loc: location of module name, if any :ivar import_loc: location of ``import`` :ivar lparen_loc: location of ``(``, if any :ivar rparen_loc: location of ``)``, if any """ _fields = ("names", "module", "level") _locs = keywordloc._locs + ("dots_loc", "module_loc", "import_loc", "lparen_loc", "rparen_loc") class Nonlocal(stmt, keywordloc): """ The ``nonlocal x, y`` statement. **Emitted since 3.0.** :ivar names: (list of string) names :ivar name_locs: locations of names """ _fields = ("names",) _locs = keywordloc._locs + ("name_locs",) class Pass(stmt, keywordloc): """The ``pass`` statement.""" class Print(stmt, keywordloc): """ The ``print >>x, y, z,`` statement. **Emitted until 3.0 or until print_function future flag is activated.** :ivar dest: (:class:`AST`) destination stream, if any :ivar values: (list of :class:`AST`) values to print :ivar nl: (boolean) whether to print newline after values :ivar dest_loc: location of ``>>`` """ _fields = ("dest", "values", "nl") _locs = keywordloc._locs + ("dest_loc",) class Raise(stmt, keywordloc): """ The ``raise exc, arg, traceback`` (2.6) or or ``raise exc from cause`` (3.0) statement. :ivar exc: (:class:`AST`) exception type or instance :ivar cause: (:class:`AST`) cause of exception, if any; **emitted since 3.0** :ivar inst: (:class:`AST`) exception instance or argument list, if any; **emitted until 3.0** :ivar tback: (:class:`AST`) traceback, if any; **emitted until 3.0** :ivar from_loc: location of ``from``, if any; **emitted since 3.0** """ _fields = ("exc", "cause", "inst", "tback") _locs = keywordloc._locs + ("from_loc",) class Return(stmt, keywordloc): """ The ``return x`` statement. :ivar value: (:class:`AST`) return value, if any """ _fields = ("value",) class Try(stmt, keywordloc): """ The ``try:· x·except y:· z·else:· t`` or ``try:· x·finally:· y`` statement. :ivar body: (list of :class:`AST`) code to try :ivar handlers: (list of :class:`ExceptHandler`) exception handlers :ivar orelse: (list of :class:`AST`) code if no exception :ivar finalbody: (list of :class:`AST`) code to finalize :ivar keyword_loc: location of ``try`` :ivar try_colon_loc: location of ``:`` after ``try`` :ivar else_loc: location of ``else`` :ivar else_colon_loc: location of ``:`` after ``else`` :ivar finally_loc: location of ``finally`` :ivar finally_colon_loc: location of ``:`` after ``finally`` """ _fields = ("body", "handlers", "orelse", "finalbody") _locs = keywordloc._locs + ("try_colon_loc", "else_loc", "else_colon_loc", "finally_loc", "finally_colon_loc",) class While(stmt, keywordloc): """ The ``while x:· y·else:· z`` statement. :ivar test: (:class:`AST`) condition :ivar body: (list of :class:`AST`) code for every iteration :ivar orelse: (list of :class:`AST`) code if empty :ivar keyword_loc: location of ``while`` :ivar while_colon_loc: location of colon after ``while`` :ivar else_loc: location of ``else``, if any :ivar else_colon_loc: location of colon after ``else``, if any """ _fields = ("test", "body", "orelse") _locs = keywordloc._locs + ("while_colon_loc", "else_loc", "else_colon_loc") class With(stmt, keywordloc): """ The ``with x as y:· z`` statement. :ivar items: (list of :class:`withitem`) bindings :ivar body: (:class:`AST`) body :ivar keyword_loc: location of ``with`` :ivar colon_loc: location of ``:`` """ _fields = ("items", "body") _locs = keywordloc._locs + ("colon_loc",) class unaryop(AST, commonloc): """Base class for unary numeric and boolean operators.""" class Invert(unaryop): """The ``~`` operator.""" class Not(unaryop): """The ``not`` operator.""" class UAdd(unaryop): """The unary ``+`` operator.""" class USub(unaryop): """The unary ``-`` operator.""" class withitem(AST, commonloc): """ The ``x as y`` clause in ``with x as y:``. :ivar context_expr: (:class:`AST`) context :ivar optional_vars: (assignable :class:`AST`) context binding, if any :ivar as_loc: location of ``as``, if any """ _fields = ("context_expr", "optional_vars") _locs = commonloc._locs + ("as_loc",) ================================================ FILE: third_party/pythonparser/diagnostic.py ================================================ """ The :mod:`Diagnostic` module concerns itself with processing and presentation of diagnostic messages. """ from __future__ import absolute_import, division, print_function, unicode_literals from functools import reduce from contextlib import contextmanager import sys, re class Diagnostic: """ A diagnostic message highlighting one or more locations in a single source buffer. :ivar level: (one of ``LEVELS``) severity level :ivar reason: (format string) diagnostic message :ivar arguments: (dictionary) substitutions for ``reason`` :ivar location: (:class:`pythonparser.source.Range`) most specific location of the problem :ivar highlights: (list of :class:`pythonparser.source.Range`) secondary locations related to the problem that are likely to be on the same line :ivar notes: (list of :class:`Diagnostic`) secondary diagnostics highlighting relevant source locations that are unlikely to be on the same line """ LEVELS = ["note", "warning", "error", "fatal"] """ Available diagnostic levels: * ``fatal`` indicates an unrecoverable error. * ``error`` indicates an error that leaves a possibility of processing more code, e.g. a recoverable parsing error. * ``warning`` indicates a potential problem. * ``note`` level diagnostics do not appear by itself, but are attached to other diagnostics to refer to and describe secondary source locations. """ def __init__(self, level, reason, arguments, location, highlights=None, notes=None): if level not in self.LEVELS: raise ValueError("level must be one of Diagnostic.LEVELS") if highlights is None: highlights = [] if notes is None: notes = [] if len(set(map(lambda x: x.source_buffer, [location] + highlights))) > 1: raise ValueError("location and highlights must refer to the same source buffer") self.level, self.reason, self.arguments = \ level, reason, arguments self.location, self.highlights, self.notes = \ location, highlights, notes def message(self): """ Returns the formatted message. """ return self.reason.format(**self.arguments) def render(self, only_line=False, colored=False): """ Returns the human-readable location of the diagnostic in the source, the formatted message, the source line corresponding to ``location`` and a line emphasizing the problematic locations in the source line using ASCII art, as a list of lines. Appends the result of calling :meth:`render` on ``notes``, if any. For example: :: :1:8-9: error: cannot add integer and string x + (1 + "a") ~ ^ ~~~ :param only_line: (bool) If true, only print line number, not line and column range """ source_line = self.location.source_line().rstrip("\n") highlight_line = bytearray(re.sub(r"[^\t]", " ", source_line), "utf-8") for hilight in self.highlights: if hilight.line() == self.location.line(): lft, rgt = hilight.column_range() highlight_line[lft:rgt] = bytearray("~", "utf-8") * (rgt - lft) lft, rgt = self.location.column_range() if rgt == lft: # Expand zero-length ranges to one ^ rgt = lft + 1 highlight_line[lft:rgt] = bytearray("^", "utf-8") * (rgt - lft) if only_line: location = "%s:%s" % (self.location.source_buffer.name, self.location.line()) else: location = str(self.location) notes = list(self.notes) if self.level != "note": expanded_location = self.location.expanded_from while expanded_location is not None: notes.insert(0, Diagnostic("note", "expanded from here", {}, self.location.expanded_from)) expanded_location = expanded_location.expanded_from rendered_notes = reduce(list.__add__, [note.render(only_line, colored) for note in notes], []) if colored: if self.level in ("error", "fatal"): level_color = 31 # red elif self.level == "warning": level_color = 35 # magenta else: # level == "note" level_color = 30 # gray return [ "\x1b[1;37m{}: \x1b[{}m{}:\x1b[37m {}\x1b[0m". format(location, level_color, self.level, self.message()), source_line, "\x1b[1;32m{}\x1b[0m".format(highlight_line.decode("utf-8")) ] + rendered_notes else: return [ "{}: {}: {}".format(location, self.level, self.message()), source_line, highlight_line.decode("utf-8") ] + rendered_notes class Error(Exception): """ :class:`Error` is an exception which carries a :class:`Diagnostic`. :ivar diagnostic: (:class:`Diagnostic`) the diagnostic """ def __init__(self, diagnostic): self.diagnostic = diagnostic def __str__(self): return "\n".join(self.diagnostic.render()) class Engine: """ :class:`Engine` is a single point through which diagnostics from lexer, parser and any AST consumer are dispatched. :ivar all_errors_are_fatal: if true, an exception is raised not only for ``fatal`` diagnostic level, but also ``error`` """ def __init__(self, all_errors_are_fatal=False): self.all_errors_are_fatal = all_errors_are_fatal self._appended_notes = [] def process(self, diagnostic): """ The default implementation of :meth:`process` renders non-fatal diagnostics to ``sys.stderr``, and raises fatal ones as a :class:`Error`. """ diagnostic.notes += self._appended_notes self.render_diagnostic(diagnostic) if diagnostic.level == "fatal" or \ (self.all_errors_are_fatal and diagnostic.level == "error"): raise Error(diagnostic) @contextmanager def context(self, *notes): """ A context manager that appends ``note`` to every diagnostic processed by this engine. """ self._appended_notes += notes yield del self._appended_notes[-len(notes):] def render_diagnostic(self, diagnostic): sys.stderr.write("\n".join(diagnostic.render()) + "\n") ================================================ FILE: third_party/pythonparser/lexer.py ================================================ """ The :mod:`lexer` module concerns itself with tokenizing Python source. """ from __future__ import absolute_import, division, print_function, unicode_literals from . import source, diagnostic import re import unicodedata import sys if sys.version_info[0] == 3: unichr = chr byte = lambda x: bytes([x]) else: byte = chr class Token: """ The :class:`Token` encapsulates a single lexer token and its location in the source code. :ivar loc: (:class:`pythonparser.source.Range`) token location :ivar kind: (string) token kind :ivar value: token value; None or a kind-specific class """ def __init__(self, loc, kind, value=None): self.loc, self.kind, self.value = loc, kind, value def __repr__(self): return "Token(%s, \"%s\", %s)" % (repr(self.loc), self.kind, repr(self.value)) class Lexer: """ The :class:`Lexer` class extracts tokens and comments from a :class:`pythonparser.source.Buffer`. :class:`Lexer` is an iterable. :ivar version: (tuple of (*major*, *minor*)) the version of Python, determining the grammar used :ivar source_buffer: (:class:`pythonparser.source.Buffer`) the source buffer :ivar diagnostic_engine: (:class:`pythonparser.diagnostic.Engine`) the diagnostic engine :ivar offset: (integer) character offset into ``source_buffer`` indicating where the next token will be recognized :ivar interactive: (boolean) whether a completely empty line should generate a NEWLINE token, for use in REPLs """ _reserved_2_6 = frozenset([ "!=", "%", "%=", "&", "&=", "(", ")", "*", "**", "**=", "*=", "+", "+=", ",", "-", "-=", ".", "/", "//", "//=", "/=", ":", ";", "<", "<<", "<<=", "<=", "<>", "=", "==", ">", ">=", ">>", ">>=", "@", "[", "]", "^", "^=", "`", "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield", "{", "|", "|=", "}", "~" ]) _reserved_3_0 = _reserved_2_6 \ - set(["<>", "`", "exec", "print"]) \ | set(["->", "...", "False", "None", "nonlocal", "True"]) _reserved_3_1 = _reserved_3_0 \ | set(["<>"]) _reserved_3_5 = _reserved_3_1 \ | set(["@", "@="]) _reserved = { (2, 6): _reserved_2_6, (2, 7): _reserved_2_6, (3, 0): _reserved_3_0, (3, 1): _reserved_3_1, (3, 2): _reserved_3_1, (3, 3): _reserved_3_1, (3, 4): _reserved_3_1, (3, 5): _reserved_3_5, } """ A map from a tuple (*major*, *minor*) corresponding to Python version to :class:`frozenset`\s of keywords. """ _string_prefixes_3_1 = frozenset(["", "r", "b", "br"]) _string_prefixes_3_3 = frozenset(["", "r", "u", "b", "br", "rb"]) # holy mother of god why _string_prefixes = { (2, 6): frozenset(["", "r", "u", "ur"]), (2, 7): frozenset(["", "r", "u", "ur", "b", "br"]), (3, 0): frozenset(["", "r", "b"]), (3, 1): _string_prefixes_3_1, (3, 2): _string_prefixes_3_1, (3, 3): _string_prefixes_3_3, (3, 4): _string_prefixes_3_3, (3, 5): _string_prefixes_3_3, } """ A map from a tuple (*major*, *minor*) corresponding to Python version to :class:`frozenset`\s of string prefixes. """ def __init__(self, source_buffer, version, diagnostic_engine, interactive=False): self.source_buffer = source_buffer self.version = version self.diagnostic_engine = diagnostic_engine self.interactive = interactive self.print_function = False self.unicode_literals = self.version >= (3, 0) self.offset = 0 self.new_line = True self.indent = [(0, source.Range(source_buffer, 0, 0), "")] self.comments = [] self.queue = [] self.parentheses = [] self.curly_braces = [] self.square_braces = [] try: reserved = self._reserved[version] except KeyError: raise NotImplementedError("pythonparser.lexer.Lexer cannot lex Python %s" % str(version)) # Sort for the regexp to obey longest-match rule. re_reserved = sorted(reserved, reverse=True, key=len) re_keywords = "|".join([kw for kw in re_reserved if kw.isalnum()]) re_operators = "|".join([re.escape(op) for op in re_reserved if not op.isalnum()]) # Python 3.0 uses ID_Start, >3.0 uses XID_Start if self.version == (3, 0): id_xid = "" else: id_xid = "X" # To speed things up on CPython, we use the re module to generate a DFA # from our token set and execute it in C. Every result yielded by # iterating this regular expression has exactly one non-empty group # that would correspond to a e.g. lex scanner branch. # The only thing left to Python code is then to select one from this # small set of groups, which is much faster than dissecting the strings. # # A lexer has to obey longest-match rule, but a regular expression does not. # Therefore, the cases in it are carefully sorted so that the longest # ones come up first. The exception is the identifier case, which would # otherwise grab all keywords; it is made to work by making it impossible # for the keyword case to match a word prefix, and ordering it before # the identifier case. self._lex_token_re = re.compile(r""" [ \t\f]* # initial whitespace ( # 1 (\\)? # ?2 line continuation ([\n]|[\r][\n]|[\r]) # 3 newline | (\#.*) # 4 comment | ( # 5 floating point or complex literal (?: [0-9]* \. [0-9]+ | [0-9]+ \.? ) [eE] [+-]? [0-9]+ | [0-9]* \. [0-9]+ | [0-9]+ \. ) ([jJ])? # ?6 complex suffix | ([0-9]+) [jJ] # 7 complex literal | (?: # integer literal ( [1-9] [0-9]* ) # 8 dec | 0[oO] ( [0-7]+ ) # 9 oct | 0[xX] ( [0-9A-Fa-f]+ ) # 10 hex | 0[bB] ( [01]+ ) # 11 bin | ( [0-9] [0-9]* ) # 12 bare oct ) ([Ll])? # 13 long option | ([BbUu]?[Rr]?) # ?14 string literal options (?: # string literal start # 15, 16, 17 long string (""\"|''') ((?: \\?[\n] | \\. | . )*?) (\15) # 18, 19, 20 short string | (" |' ) ((?: \\ [\n] | \\. | . )*?) (\18) # 21 unterminated | (""\"|'''|"|') ) | ((?:{keywords})\b|{operators}) # 22 keywords and operators | ([A-Za-z_][A-Za-z0-9_]*\b) # 23 identifier | (\p{{{id_xid}ID_Start}}\p{{{id_xid}ID_Continue}}*) # 24 Unicode identifier | ($) # 25 end-of-file ) """.format(keywords=re_keywords, operators=re_operators, id_xid=id_xid), re.VERBOSE|re.UNICODE) # These are identical for all lexer instances. _lex_escape_pattern = r""" \\(?: ([\n\\'"abfnrtv]) # 1 single-char | ([0-7]{1,3}) # 2 oct | x([0-9A-Fa-f]{2}) # 3 hex ) """ _lex_escape_re = re.compile(_lex_escape_pattern.encode(), re.VERBOSE) _lex_escape_unicode_re = re.compile(_lex_escape_pattern + r""" | \\(?: u([0-9A-Fa-f]{4}) # 4 unicode-16 | U([0-9A-Fa-f]{8}) # 5 unicode-32 | N\{(.+?)\} # 6 unicode-name ) """, re.VERBOSE) def next(self, eof_token=False): """ Returns token at ``offset`` as a :class:`Token` and advances ``offset`` to point past the end of the token, where the token has: - *range* which is a :class:`pythonparser.source.Range` that includes the token but not surrounding whitespace, - *kind* which is a string containing one of Python keywords or operators, ``newline``, ``float``, ``int``, ``complex``, ``strbegin``, ``strdata``, ``strend``, ``ident``, ``indent``, ``dedent`` or ``eof`` (if ``eof_token`` is True). - *value* which is the flags as lowercase string if *kind* is ``strbegin``, the string contents if *kind* is ``strdata``, the numeric value if *kind* is ``float``, ``int`` or ``complex``, the identifier if *kind* is ``ident`` and ``None`` in any other case. :param eof_token: if true, will return a token with kind ``eof`` when the input is exhausted; if false, will raise ``StopIteration``. """ if len(self.queue) == 0: self._refill(eof_token) return self.queue.pop(0) def peek(self, eof_token=False): """Same as :meth:`next`, except the token is not dequeued.""" if len(self.queue) == 0: self._refill(eof_token) return self.queue[-1] # We need separate next and _refill because lexing can sometimes # generate several tokens, e.g. INDENT def _refill(self, eof_token): if self.offset == len(self.source_buffer.source): range = source.Range(self.source_buffer, self.offset, self.offset) if not self.new_line: self.new_line = True self.queue.append(Token(range, "newline")) return for i in self.indent[1:]: self.indent.pop(-1) self.queue.append(Token(range, "dedent")) if eof_token: self.queue.append(Token(range, "eof")) elif len(self.queue) == 0: raise StopIteration return match = self._lex_token_re.match(self.source_buffer.source, self.offset) if match is None: diag = diagnostic.Diagnostic( "fatal", "unexpected {character}", {"character": repr(self.source_buffer.source[self.offset]).lstrip("u")}, source.Range(self.source_buffer, self.offset, self.offset + 1)) self.diagnostic_engine.process(diag) # Should we emit indent/dedent? if self.new_line and \ match.group(3) is None and \ match.group(4) is None: # not a blank line whitespace = match.string[match.start(0):match.start(1)] level = len(whitespace.expandtabs()) range = source.Range(self.source_buffer, match.start(1), match.start(1)) if level > self.indent[-1][0]: self.indent.append((level, range, whitespace)) self.queue.append(Token(range, "indent")) elif level < self.indent[-1][0]: exact = False while level <= self.indent[-1][0]: if level == self.indent[-1][0] or self.indent[-1][0] == 0: exact = True break self.indent.pop(-1) self.queue.append(Token(range, "dedent")) if not exact: note = diagnostic.Diagnostic( "note", "expected to match level here", {}, self.indent[-1][1]) error = diagnostic.Diagnostic( "fatal", "inconsistent indentation", {}, range, notes=[note]) self.diagnostic_engine.process(error) elif whitespace != self.indent[-1][2] and self.version >= (3, 0): error = diagnostic.Diagnostic( "error", "inconsistent use of tabs and spaces in indentation", {}, range) self.diagnostic_engine.process(error) # Prepare for next token. self.offset = match.end(0) tok_range = source.Range(self.source_buffer, *match.span(1)) if match.group(3) is not None: # newline if len(self.parentheses) + len(self.square_braces) + len(self.curly_braces) > 0: # 2.1.6 Implicit line joining return self._refill(eof_token) if match.group(2) is not None: # 2.1.5. Explicit line joining return self._refill(eof_token) if self.new_line and not \ (self.interactive and match.group(0) == match.group(3)): # REPL terminator # 2.1.7. Blank lines return self._refill(eof_token) self.new_line = True self.queue.append(Token(tok_range, "newline")) return if match.group(4) is not None: # comment self.comments.append(source.Comment(tok_range, match.group(4))) return self._refill(eof_token) # Lexing non-whitespace now. self.new_line = False if sys.version_info > (3,) or not match.group(13): int_type = int else: int_type = long if match.group(5) is not None: # floating point or complex literal if match.group(6) is None: self.queue.append(Token(tok_range, "float", float(match.group(5)))) else: self.queue.append(Token(tok_range, "complex", float(match.group(5)) * 1j)) elif match.group(7) is not None: # complex literal self.queue.append(Token(tok_range, "complex", int(match.group(7)) * 1j)) elif match.group(8) is not None: # integer literal, dec literal = match.group(8) self._check_long_literal(tok_range, match.group(1)) self.queue.append(Token(tok_range, "int", int_type(literal))) elif match.group(9) is not None: # integer literal, oct literal = match.group(9) self._check_long_literal(tok_range, match.group(1)) self.queue.append(Token(tok_range, "int", int_type(literal, 8))) elif match.group(10) is not None: # integer literal, hex literal = match.group(10) self._check_long_literal(tok_range, match.group(1)) self.queue.append(Token(tok_range, "int", int_type(literal, 16))) elif match.group(11) is not None: # integer literal, bin literal = match.group(11) self._check_long_literal(tok_range, match.group(1)) self.queue.append(Token(tok_range, "int", int_type(literal, 2))) elif match.group(12) is not None: # integer literal, bare oct literal = match.group(12) if len(literal) > 1 and self.version >= (3, 0): error = diagnostic.Diagnostic( "error", "in Python 3, decimal literals must not start with a zero", {}, source.Range(self.source_buffer, tok_range.begin_pos, tok_range.begin_pos + 1)) self.diagnostic_engine.process(error) self.queue.append(Token(tok_range, "int", int(literal, 8))) elif match.group(15) is not None: # long string literal self._string_literal( options=match.group(14), begin_span=(match.start(14), match.end(15)), data=match.group(16), data_span=match.span(16), end_span=match.span(17)) elif match.group(18) is not None: # short string literal self._string_literal( options=match.group(14), begin_span=(match.start(14), match.end(18)), data=match.group(19), data_span=match.span(19), end_span=match.span(20)) elif match.group(21) is not None: # unterminated string error = diagnostic.Diagnostic( "fatal", "unterminated string", {}, tok_range) self.diagnostic_engine.process(error) elif match.group(22) is not None: # keywords and operators kwop = match.group(22) self._match_pair_delim(tok_range, kwop) if kwop == "print" and self.print_function: self.queue.append(Token(tok_range, "ident", "print")) else: self.queue.append(Token(tok_range, kwop)) elif match.group(23) is not None: # identifier self.queue.append(Token(tok_range, "ident", match.group(23))) elif match.group(24) is not None: # Unicode identifier if self.version < (3, 0): error = diagnostic.Diagnostic( "error", "in Python 2, Unicode identifiers are not allowed", {}, tok_range) self.diagnostic_engine.process(error) self.queue.append(Token(tok_range, "ident", match.group(24))) elif match.group(25) is not None: # end-of-file # Reuse the EOF logic return self._refill(eof_token) else: assert False def _string_literal(self, options, begin_span, data, data_span, end_span): options = options.lower() begin_range = source.Range(self.source_buffer, *begin_span) data_range = source.Range(self.source_buffer, *data_span) if options not in self._string_prefixes[self.version]: error = diagnostic.Diagnostic( "error", "string prefix '{prefix}' is not available in Python {major}.{minor}", {"prefix": options, "major": self.version[0], "minor": self.version[1]}, begin_range) self.diagnostic_engine.process(error) self.queue.append(Token(begin_range, "strbegin", options)) self.queue.append(Token(data_range, "strdata", self._replace_escape(data_range, options, data))) self.queue.append(Token(source.Range(self.source_buffer, *end_span), "strend")) def _replace_escape(self, range, mode, value): is_raw = ("r" in mode) is_unicode = "u" in mode or ("b" not in mode and self.unicode_literals) if not is_unicode: value = value.encode(self.source_buffer.encoding) if is_raw: return value return self._replace_escape_bytes(value) if is_raw: return value return self._replace_escape_unicode(range, value) def _replace_escape_unicode(self, range, value): chunks = [] offset = 0 while offset < len(value): match = self._lex_escape_unicode_re.search(value, offset) if match is None: # Append the remaining of the string chunks.append(value[offset:]) break # Append the part of string before match chunks.append(value[offset:match.start()]) offset = match.end() # Process the escape if match.group(1) is not None: # single-char chr = match.group(1) if chr == "\n": pass elif chr == "\\" or chr == "'" or chr == "\"": chunks.append(chr) elif chr == "a": chunks.append("\a") elif chr == "b": chunks.append("\b") elif chr == "f": chunks.append("\f") elif chr == "n": chunks.append("\n") elif chr == "r": chunks.append("\r") elif chr == "t": chunks.append("\t") elif chr == "v": chunks.append("\v") elif match.group(2) is not None: # oct chunks.append(unichr(int(match.group(2), 8))) elif match.group(3) is not None: # hex chunks.append(unichr(int(match.group(3), 16))) elif match.group(4) is not None: # unicode-16 chunks.append(unichr(int(match.group(4), 16))) elif match.group(5) is not None: # unicode-32 try: chunks.append(unichr(int(match.group(5), 16))) except ValueError: error = diagnostic.Diagnostic( "error", "unicode character out of range", {}, source.Range(self.source_buffer, range.begin_pos + match.start(0), range.begin_pos + match.end(0))) self.diagnostic_engine.process(error) elif match.group(6) is not None: # unicode-name try: chunks.append(unicodedata.lookup(match.group(6))) except KeyError: error = diagnostic.Diagnostic( "error", "unknown unicode character name", {}, source.Range(self.source_buffer, range.begin_pos + match.start(0), range.begin_pos + match.end(0))) self.diagnostic_engine.process(error) return "".join(chunks) def _replace_escape_bytes(self, value): chunks = [] offset = 0 while offset < len(value): match = self._lex_escape_re.search(value, offset) if match is None: # Append the remaining of the string chunks.append(value[offset:]) break # Append the part of string before match chunks.append(value[offset:match.start()]) offset = match.end() # Process the escape if match.group(1) is not None: # single-char chr = match.group(1) if chr == b"\n": pass elif chr == b"\\" or chr == b"'" or chr == b"\"": chunks.append(chr) elif chr == b"a": chunks.append(b"\a") elif chr == b"b": chunks.append(b"\b") elif chr == b"f": chunks.append(b"\f") elif chr == b"n": chunks.append(b"\n") elif chr == b"r": chunks.append(b"\r") elif chr == b"t": chunks.append(b"\t") elif chr == b"v": chunks.append(b"\v") elif match.group(2) is not None: # oct chunks.append(byte(int(match.group(2), 8))) elif match.group(3) is not None: # hex chunks.append(byte(int(match.group(3), 16))) return b"".join(chunks) def _check_long_literal(self, range, literal): if literal[-1] in "lL" and self.version >= (3, 0): error = diagnostic.Diagnostic( "error", "in Python 3, long integer literals were removed", {}, source.Range(self.source_buffer, range.end_pos - 1, range.end_pos)) self.diagnostic_engine.process(error) def _match_pair_delim(self, range, kwop): if kwop == "(": self.parentheses.append(range) elif kwop == "[": self.square_braces.append(range) elif kwop == "{": self.curly_braces.append(range) elif kwop == ")": self._check_innermost_pair_delim(range, "(") self.parentheses.pop() elif kwop == "]": self._check_innermost_pair_delim(range, "[") self.square_braces.pop() elif kwop == "}": self._check_innermost_pair_delim(range, "{") self.curly_braces.pop() def _check_innermost_pair_delim(self, range, expected): ranges = [] if len(self.parentheses) > 0: ranges.append(("(", self.parentheses[-1])) if len(self.square_braces) > 0: ranges.append(("[", self.square_braces[-1])) if len(self.curly_braces) > 0: ranges.append(("{", self.curly_braces[-1])) ranges.sort(key=lambda k: k[1].begin_pos) if any(ranges): compl_kind, compl_range = ranges[-1] if compl_kind != expected: note = diagnostic.Diagnostic( "note", "'{delimiter}' opened here", {"delimiter": compl_kind}, compl_range) error = diagnostic.Diagnostic( "fatal", "mismatched '{delimiter}'", {"delimiter": range.source()}, range, notes=[note]) self.diagnostic_engine.process(error) else: error = diagnostic.Diagnostic( "fatal", "mismatched '{delimiter}'", {"delimiter": range.source()}, range) self.diagnostic_engine.process(error) def __iter__(self): return self def __next__(self): return self.next() ================================================ FILE: third_party/pythonparser/parser.py ================================================ # encoding:utf-8 """ The :mod:`parser` module concerns itself with parsing Python source. """ from __future__ import absolute_import, division, print_function, unicode_literals from functools import reduce from . import source, diagnostic, lexer, ast # A few notes about our approach to parsing: # # Python uses an LL(1) parser generator. It's a bit weird, because # the usual reason to choose LL(1) is to make a handwritten parser # possible, however Python's grammar is formulated in a way that # is much more easily recognized if you make an FSM rather than # the usual "if accept(token)..." ladder. So in a way it is # the worst of both worlds. # # We don't use a parser generator because we want to have an unified # grammar for all Python versions, and also have grammar coverage # analysis and nice error recovery. To make the grammar compact, # we use combinators to compose it from predefined fragments, # such as "sequence" or "alternation" or "Kleene star". This easily # gives us one token of lookahead in most cases, but e.g. not # in the following one: # # argument: test | test '=' test # # There are two issues with this. First, in an alternation, the first # variant will be tried (and accepted) earlier. Second, if we reverse # them, by the point it is clear ``'='`` will not be accepted, ``test`` # has already been consumed. # # The way we fix this is by reordering rules so that longest match # comes first, and adding backtracking on alternations (as well as # plus and star, since those have a hidden alternation inside). # # While backtracking can in principle make asymptotical complexity # worse, it never makes parsing syntactically correct code supralinear # with Python's LL(1) grammar, and we could not come up with any # pathological incorrect input as well. # Coverage data _all_rules = [] _all_stmts = {} # Generic LL parsing combinators class Unmatched: pass unmatched = Unmatched() def llrule(loc, expected, cases=1): if loc is None: def decorator(rule): rule.expected = expected return rule else: def decorator(inner_rule): if cases == 1: def rule(*args, **kwargs): result = inner_rule(*args, **kwargs) if result is not unmatched: rule.covered[0] = True return result else: rule = inner_rule rule.loc, rule.expected, rule.covered = \ loc, expected, [False] * cases _all_rules.append(rule) return rule return decorator def action(inner_rule, loc=None): """ A decorator returning a function that first runs ``inner_rule`` and then, if its return value is not None, maps that value using ``mapper``. If the value being mapped is a tuple, it is expanded into multiple arguments. Similar to attaching semantic actions to rules in traditional parser generators. """ def decorator(mapper): @llrule(loc, inner_rule.expected) def outer_rule(parser): result = inner_rule(parser) if result is unmatched: return result if isinstance(result, tuple): return mapper(parser, *result) else: return mapper(parser, result) return outer_rule return decorator def Eps(value=None, loc=None): """A rule that accepts no tokens (epsilon) and returns ``value``.""" @llrule(loc, lambda parser: []) def rule(parser): return value return rule def Tok(kind, loc=None): """A rule that accepts a token of kind ``kind`` and returns it, or returns None.""" @llrule(loc, lambda parser: [kind]) def rule(parser): return parser._accept(kind) return rule def Loc(kind, loc=None): """A rule that accepts a token of kind ``kind`` and returns its location, or returns None.""" @llrule(loc, lambda parser: [kind]) def rule(parser): result = parser._accept(kind) if result is unmatched: return result return result.loc return rule def Rule(name, loc=None): """A proxy for a rule called ``name`` which may not be yet defined.""" @llrule(loc, lambda parser: getattr(parser, name).expected(parser)) def rule(parser): return getattr(parser, name)() return rule def Expect(inner_rule, loc=None): """A rule that executes ``inner_rule`` and emits a diagnostic error if it returns None.""" @llrule(loc, inner_rule.expected) def rule(parser): result = inner_rule(parser) if result is unmatched: expected = reduce(list.__add__, [rule.expected(parser) for rule in parser._errrules]) expected = list(sorted(set(expected))) if len(expected) > 1: expected = " or ".join([", ".join(expected[0:-1]), expected[-1]]) elif len(expected) == 1: expected = expected[0] else: expected = "(impossible)" error_tok = parser._tokens[parser._errindex] error = diagnostic.Diagnostic( "fatal", "unexpected {actual}: expected {expected}", {"actual": error_tok.kind, "expected": expected}, error_tok.loc) parser.diagnostic_engine.process(error) return result return rule def Seq(first_rule, *rest_of_rules, **kwargs): """ A rule that accepts a sequence of tokens satisfying ``rules`` and returns a tuple containing their return values, or None if the first rule was not satisfied. """ @llrule(kwargs.get("loc", None), first_rule.expected) def rule(parser): result = first_rule(parser) if result is unmatched: return result results = [result] for rule in rest_of_rules: result = rule(parser) if result is unmatched: return result results.append(result) return tuple(results) return rule def SeqN(n, *inner_rules, **kwargs): """ A rule that accepts a sequence of tokens satisfying ``rules`` and returns the value returned by rule number ``n``, or None if the first rule was not satisfied. """ @action(Seq(*inner_rules), loc=kwargs.get("loc", None)) def rule(parser, *values): return values[n] return rule def Alt(*inner_rules, **kwargs): """ A rule that expects a sequence of tokens satisfying one of ``rules`` in sequence (a rule is satisfied when it returns anything but None) and returns the return value of that rule, or None if no rules were satisfied. """ loc = kwargs.get("loc", None) expected = lambda parser: reduce(list.__add__, map(lambda x: x.expected(parser), inner_rules)) if loc is not None: @llrule(loc, expected, cases=len(inner_rules)) def rule(parser): data = parser._save() for idx, inner_rule in enumerate(inner_rules): result = inner_rule(parser) if result is unmatched: parser._restore(data, rule=inner_rule) else: rule.covered[idx] = True return result return unmatched else: @llrule(loc, expected, cases=len(inner_rules)) def rule(parser): data = parser._save() for inner_rule in inner_rules: result = inner_rule(parser) if result is unmatched: parser._restore(data, rule=inner_rule) else: return result return unmatched return rule def Opt(inner_rule, loc=None): """Shorthand for ``Alt(inner_rule, Eps())``""" return Alt(inner_rule, Eps(), loc=loc) def Star(inner_rule, loc=None): """ A rule that accepts a sequence of tokens satisfying ``inner_rule`` zero or more times, and returns the returned values in a :class:`list`. """ @llrule(loc, lambda parser: []) def rule(parser): results = [] while True: data = parser._save() result = inner_rule(parser) if result is unmatched: parser._restore(data, rule=inner_rule) return results results.append(result) return rule def Plus(inner_rule, loc=None): """ A rule that accepts a sequence of tokens satisfying ``inner_rule`` one or more times, and returns the returned values in a :class:`list`. """ @llrule(loc, inner_rule.expected) def rule(parser): result = inner_rule(parser) if result is unmatched: return result results = [result] while True: data = parser._save() result = inner_rule(parser) if result is unmatched: parser._restore(data, rule=inner_rule) return results results.append(result) return rule class commalist(list): __slots__ = ("trailing_comma",) def List(inner_rule, separator_tok, trailing, leading=True, loc=None): if not trailing: @action(Seq(inner_rule, Star(SeqN(1, Tok(separator_tok), inner_rule))), loc=loc) def outer_rule(parser, first, rest): return [first] + rest return outer_rule else: # A rule like this: stmt (';' stmt)* [';'] # This doesn't yield itself to combinators above, because disambiguating # another iteration of the Kleene star and the trailing separator # requires two lookahead tokens (naively). separator_rule = Tok(separator_tok) @llrule(loc, inner_rule.expected) def rule(parser): results = commalist() if leading: result = inner_rule(parser) if result is unmatched: return result else: results.append(result) while True: result = separator_rule(parser) if result is unmatched: results.trailing_comma = None return results result_1 = inner_rule(parser) if result_1 is unmatched: results.trailing_comma = result return results else: results.append(result_1) return rule # Python AST specific parser combinators def Newline(loc=None): """A rule that accepts token of kind ``newline`` and returns an empty list.""" @llrule(loc, lambda parser: ["newline"]) def rule(parser): result = parser._accept("newline") if result is unmatched: return result return [] return rule def Oper(klass, *kinds, **kwargs): """ A rule that accepts a sequence of tokens of kinds ``kinds`` and returns an instance of ``klass`` with ``loc`` encompassing the entire sequence or None if the first token is not of ``kinds[0]``. """ @action(Seq(*map(Loc, kinds)), loc=kwargs.get("loc", None)) def rule(parser, *tokens): return klass(loc=tokens[0].join(tokens[-1])) return rule def BinOper(expr_rulename, op_rule, node=ast.BinOp, loc=None): @action(Seq(Rule(expr_rulename), Star(Seq(op_rule, Rule(expr_rulename)))), loc=loc) def rule(parser, lhs, trailers): for (op, rhs) in trailers: lhs = node(left=lhs, op=op, right=rhs, loc=lhs.loc.join(rhs.loc)) return lhs return rule def BeginEnd(begin_tok, inner_rule, end_tok, empty=None, loc=None): @action(Seq(Loc(begin_tok), inner_rule, Loc(end_tok)), loc=loc) def rule(parser, begin_loc, node, end_loc): if node is None: node = empty(parser) # Collection nodes don't have loc yet. If a node has loc at this # point, it means it's an expression passed in parentheses. if node.loc is None and type(node) in [ ast.List, ast.ListComp, ast.Dict, ast.DictComp, ast.Set, ast.SetComp, ast.GeneratorExp, ast.Tuple, ast.Repr, ast.Call, ast.Subscript, ast.arguments]: node.begin_loc, node.end_loc, node.loc = \ begin_loc, end_loc, begin_loc.join(end_loc) return node return rule class Parser: # Generic LL parsing methods def __init__(self, lexer, version, diagnostic_engine): self._init_version(version) self.diagnostic_engine = diagnostic_engine self.lexer = lexer self._tokens = [] self._index = -1 self._errindex = -1 self._errrules = [] self._advance() def _save(self): return self._index def _restore(self, data, rule): self._index = data self._token = self._tokens[self._index] if self._index > self._errindex: # We have advanced since last error self._errindex = self._index self._errrules = [rule] elif self._index == self._errindex: # We're at the same place as last error self._errrules.append(rule) else: # We've backtracked far and are now just failing the # whole parse pass def _advance(self): self._index += 1 if self._index == len(self._tokens): self._tokens.append(self.lexer.next(eof_token=True)) self._token = self._tokens[self._index] def _accept(self, expected_kind): if self._token.kind == expected_kind: result = self._token self._advance() return result return unmatched # Python-specific methods def _init_version(self, version): if version in ((2, 6), (2, 7)): if version == (2, 6): self.with_stmt = self.with_stmt__26 self.atom_6 = self.atom_6__26 else: self.with_stmt = self.with_stmt__27 self.atom_6 = self.atom_6__27 self.except_clause_1 = self.except_clause_1__26 self.classdef = self.classdef__26 self.subscript = self.subscript__26 self.raise_stmt = self.raise_stmt__26 self.comp_if = self.comp_if__26 self.atom = self.atom__26 self.funcdef = self.funcdef__26 self.parameters = self.parameters__26 self.varargslist = self.varargslist__26 self.comparison_1 = self.comparison_1__26 self.exprlist_1 = self.exprlist_1__26 self.testlist_comp_1 = self.testlist_comp_1__26 self.expr_stmt_1 = self.expr_stmt_1__26 self.yield_expr = self.yield_expr__26 return elif version in ((3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5)): if version == (3, 0): self.with_stmt = self.with_stmt__26 # lol else: self.with_stmt = self.with_stmt__27 self.except_clause_1 = self.except_clause_1__30 self.classdef = self.classdef__30 self.subscript = self.subscript__30 self.raise_stmt = self.raise_stmt__30 self.comp_if = self.comp_if__30 self.atom = self.atom__30 self.funcdef = self.funcdef__30 self.parameters = self.parameters__30 if version < (3, 2): self.varargslist = self.varargslist__30 self.typedargslist = self.typedargslist__30 self.comparison_1 = self.comparison_1__30 self.star_expr = self.star_expr__30 self.exprlist_1 = self.exprlist_1__30 self.testlist_comp_1 = self.testlist_comp_1__26 self.expr_stmt_1 = self.expr_stmt_1__26 else: self.varargslist = self.varargslist__32 self.typedargslist = self.typedargslist__32 self.comparison_1 = self.comparison_1__32 self.star_expr = self.star_expr__32 self.exprlist_1 = self.exprlist_1__32 self.testlist_comp_1 = self.testlist_comp_1__32 self.expr_stmt_1 = self.expr_stmt_1__32 if version < (3, 3): self.yield_expr = self.yield_expr__26 else: self.yield_expr = self.yield_expr__33 return raise NotImplementedError("pythonparser.parser.Parser cannot parse Python %s" % str(version)) def _arguments(self, args=None, defaults=None, kwonlyargs=None, kw_defaults=None, vararg=None, kwarg=None, star_loc=None, dstar_loc=None, begin_loc=None, end_loc=None, equals_locs=None, kw_equals_locs=None, loc=None): if args is None: args = [] if defaults is None: defaults = [] if kwonlyargs is None: kwonlyargs = [] if kw_defaults is None: kw_defaults = [] if equals_locs is None: equals_locs = [] if kw_equals_locs is None: kw_equals_locs = [] return ast.arguments(args=args, defaults=defaults, kwonlyargs=kwonlyargs, kw_defaults=kw_defaults, vararg=vararg, kwarg=kwarg, star_loc=star_loc, dstar_loc=dstar_loc, begin_loc=begin_loc, end_loc=end_loc, equals_locs=equals_locs, kw_equals_locs=kw_equals_locs, loc=loc) def _arg(self, tok, colon_loc=None, annotation=None): loc = tok.loc if annotation: loc = loc.join(annotation.loc) return ast.arg(arg=tok.value, annotation=annotation, arg_loc=tok.loc, colon_loc=colon_loc, loc=loc) def _empty_arglist(self): return ast.Call(args=[], keywords=[], starargs=None, kwargs=None, star_loc=None, dstar_loc=None, loc=None) def _wrap_tuple(self, elts): assert len(elts) > 0 if len(elts) > 1: return ast.Tuple(ctx=None, elts=elts, loc=elts[0].loc.join(elts[-1].loc), begin_loc=None, end_loc=None) else: return elts[0] def _assignable(self, node, is_delete=False): if isinstance(node, ast.Name) or isinstance(node, ast.Subscript) or \ isinstance(node, ast.Attribute) or isinstance(node, ast.Starred): return node elif (isinstance(node, ast.List) or isinstance(node, ast.Tuple)) and \ any(node.elts): node.elts = [self._assignable(elt, is_delete) for elt in node.elts] return node else: if is_delete: error = diagnostic.Diagnostic( "fatal", "cannot delete this expression", {}, node.loc) else: error = diagnostic.Diagnostic( "fatal", "cannot assign to this expression", {}, node.loc) self.diagnostic_engine.process(error) def add_flags(self, flags): if "print_function" in flags: self.lexer.print_function = True if "unicode_literals" in flags: self.lexer.unicode_literals = True # Grammar @action(Expect(Alt(Newline(), Rule("simple_stmt"), SeqN(0, Rule("compound_stmt"), Newline())))) def single_input(self, body): """single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE""" loc = None if body != []: loc = body[0].loc return ast.Interactive(body=body, loc=loc) @action(Expect(SeqN(0, Star(Alt(Newline(), Rule("stmt"))), Tok("eof")))) def file_input(parser, body): """file_input: (NEWLINE | stmt)* ENDMARKER""" body = reduce(list.__add__, body, []) loc = None if body != []: loc = body[0].loc return ast.Module(body=body, loc=loc) @action(Expect(SeqN(0, Rule("testlist"), Star(Tok("newline")), Tok("eof")))) def eval_input(self, expr): """eval_input: testlist NEWLINE* ENDMARKER""" return ast.Expression(body=[expr], loc=expr.loc) @action(Seq(Loc("@"), List(Tok("ident"), ".", trailing=False), Opt(BeginEnd("(", Opt(Rule("arglist")), ")", empty=_empty_arglist)), Loc("newline"))) def decorator(self, at_loc, idents, call_opt, newline_loc): """decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE""" root = idents[0] dec_loc = root.loc expr = ast.Name(id=root.value, ctx=None, loc=root.loc) for ident in idents[1:]: dot_loc = ident.loc.begin() dot_loc.begin_pos -= 1 dec_loc = dec_loc.join(ident.loc) expr = ast.Attribute(value=expr, attr=ident.value, ctx=None, loc=expr.loc.join(ident.loc), attr_loc=ident.loc, dot_loc=dot_loc) if call_opt: call_opt.func = expr call_opt.loc = dec_loc.join(call_opt.loc) expr = call_opt return at_loc, expr decorators = Plus(Rule("decorator")) """decorators: decorator+""" @action(Seq(Rule("decorators"), Alt(Rule("classdef"), Rule("funcdef")))) def decorated(self, decorators, classfuncdef): """decorated: decorators (classdef | funcdef)""" classfuncdef.at_locs = list(map(lambda x: x[0], decorators)) classfuncdef.decorator_list = list(map(lambda x: x[1], decorators)) classfuncdef.loc = classfuncdef.loc.join(decorators[0][0]) return classfuncdef @action(Seq(Loc("def"), Tok("ident"), Rule("parameters"), Loc(":"), Rule("suite"))) def funcdef__26(self, def_loc, ident_tok, args, colon_loc, suite): """(2.6, 2.7) funcdef: 'def' NAME parameters ':' suite""" return ast.FunctionDef(name=ident_tok.value, args=args, returns=None, body=suite, decorator_list=[], at_locs=[], keyword_loc=def_loc, name_loc=ident_tok.loc, colon_loc=colon_loc, arrow_loc=None, loc=def_loc.join(suite[-1].loc)) @action(Seq(Loc("def"), Tok("ident"), Rule("parameters"), Opt(Seq(Loc("->"), Rule("test"))), Loc(":"), Rule("suite"))) def funcdef__30(self, def_loc, ident_tok, args, returns_opt, colon_loc, suite): """(3.0-) funcdef: 'def' NAME parameters ['->' test] ':' suite""" arrow_loc = returns = None if returns_opt: arrow_loc, returns = returns_opt return ast.FunctionDef(name=ident_tok.value, args=args, returns=returns, body=suite, decorator_list=[], at_locs=[], keyword_loc=def_loc, name_loc=ident_tok.loc, colon_loc=colon_loc, arrow_loc=arrow_loc, loc=def_loc.join(suite[-1].loc)) parameters__26 = BeginEnd("(", Opt(Rule("varargslist")), ")", empty=_arguments) """(2.6, 2.7) parameters: '(' [varargslist] ')'""" parameters__30 = BeginEnd("(", Opt(Rule("typedargslist")), ")", empty=_arguments) """(3.0) parameters: '(' [typedargslist] ')'""" varargslist__26_1 = Seq(Rule("fpdef"), Opt(Seq(Loc("="), Rule("test")))) @action(Seq(Loc("**"), Tok("ident"))) def varargslist__26_2(self, dstar_loc, kwarg_tok): return self._arguments(kwarg=self._arg(kwarg_tok), dstar_loc=dstar_loc, loc=dstar_loc.join(kwarg_tok.loc)) @action(Seq(Loc("*"), Tok("ident"), Opt(Seq(Tok(","), Loc("**"), Tok("ident"))))) def varargslist__26_3(self, star_loc, vararg_tok, kwarg_opt): dstar_loc = kwarg = None loc = star_loc.join(vararg_tok.loc) vararg = self._arg(vararg_tok) if kwarg_opt: _, dstar_loc, kwarg_tok = kwarg_opt kwarg = self._arg(kwarg_tok) loc = star_loc.join(kwarg_tok.loc) return self._arguments(vararg=vararg, kwarg=kwarg, star_loc=star_loc, dstar_loc=dstar_loc, loc=loc) @action(Eps(value=())) def varargslist__26_4(self): return self._arguments() @action(Alt(Seq(Star(SeqN(0, varargslist__26_1, Tok(","))), Alt(varargslist__26_2, varargslist__26_3)), Seq(List(varargslist__26_1, ",", trailing=True), varargslist__26_4))) def varargslist__26(self, fparams, args): """ (2.6, 2.7) varargslist: ((fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [',']) """ for fparam, default_opt in fparams: if default_opt: equals_loc, default = default_opt args.equals_locs.append(equals_loc) args.defaults.append(default) elif len(args.defaults) > 0: error = diagnostic.Diagnostic( "fatal", "non-default argument follows default argument", {}, fparam.loc, [args.args[-1].loc.join(args.defaults[-1].loc)]) self.diagnostic_engine.process(error) args.args.append(fparam) def fparam_loc(fparam, default_opt): if default_opt: equals_loc, default = default_opt return fparam.loc.join(default.loc) else: return fparam.loc if args.loc is None: args.loc = fparam_loc(*fparams[0]).join(fparam_loc(*fparams[-1])) elif len(fparams) > 0: args.loc = args.loc.join(fparam_loc(*fparams[0])) return args @action(Tok("ident")) def fpdef_1(self, ident_tok): return ast.arg(arg=ident_tok.value, annotation=None, arg_loc=ident_tok.loc, colon_loc=None, loc=ident_tok.loc) fpdef = Alt(fpdef_1, BeginEnd("(", Rule("fplist"), ")", empty=lambda self: ast.Tuple(elts=[], ctx=None, loc=None))) """fpdef: NAME | '(' fplist ')'""" def _argslist(fpdef_rule, old_style=False): argslist_1 = Seq(fpdef_rule, Opt(Seq(Loc("="), Rule("test")))) @action(Seq(Loc("**"), Tok("ident"))) def argslist_2(self, dstar_loc, kwarg_tok): return self._arguments(kwarg=self._arg(kwarg_tok), dstar_loc=dstar_loc, loc=dstar_loc.join(kwarg_tok.loc)) @action(Seq(Loc("*"), Tok("ident"), Star(SeqN(1, Tok(","), argslist_1)), Opt(Seq(Tok(","), Loc("**"), Tok("ident"))))) def argslist_3(self, star_loc, vararg_tok, fparams, kwarg_opt): dstar_loc = kwarg = None loc = star_loc.join(vararg_tok.loc) vararg = self._arg(vararg_tok) if kwarg_opt: _, dstar_loc, kwarg_tok = kwarg_opt kwarg = self._arg(kwarg_tok) loc = star_loc.join(kwarg_tok.loc) kwonlyargs, kw_defaults, kw_equals_locs = [], [], [] for fparam, default_opt in fparams: if default_opt: equals_loc, default = default_opt kw_equals_locs.append(equals_loc) kw_defaults.append(default) else: kw_defaults.append(None) kwonlyargs.append(fparam) if any(kw_defaults): loc = loc.join(kw_defaults[-1].loc) elif any(kwonlyargs): loc = loc.join(kwonlyargs[-1].loc) return self._arguments(vararg=vararg, kwarg=kwarg, kwonlyargs=kwonlyargs, kw_defaults=kw_defaults, star_loc=star_loc, dstar_loc=dstar_loc, kw_equals_locs=kw_equals_locs, loc=loc) argslist_4 = Alt(argslist_2, argslist_3) @action(Eps(value=())) def argslist_5(self): return self._arguments() if old_style: argslist = Alt(Seq(Star(SeqN(0, argslist_1, Tok(","))), argslist_4), Seq(List(argslist_1, ",", trailing=True), argslist_5)) else: argslist = Alt(Seq(Eps(value=[]), argslist_4), Seq(List(argslist_1, ",", trailing=False), Alt(SeqN(1, Tok(","), Alt(argslist_4, argslist_5)), argslist_5))) def argslist_action(self, fparams, args): for fparam, default_opt in fparams: if default_opt: equals_loc, default = default_opt args.equals_locs.append(equals_loc) args.defaults.append(default) elif len(args.defaults) > 0: error = diagnostic.Diagnostic( "fatal", "non-default argument follows default argument", {}, fparam.loc, [args.args[-1].loc.join(args.defaults[-1].loc)]) self.diagnostic_engine.process(error) args.args.append(fparam) def fparam_loc(fparam, default_opt): if default_opt: equals_loc, default = default_opt return fparam.loc.join(default.loc) else: return fparam.loc if args.loc is None: args.loc = fparam_loc(*fparams[0]).join(fparam_loc(*fparams[-1])) elif len(fparams) > 0: args.loc = args.loc.join(fparam_loc(*fparams[0])) return args return action(argslist)(argslist_action) typedargslist__30 = _argslist(Rule("tfpdef"), old_style=True) """ (3.0, 3.1) typedargslist: ((tfpdef ['=' test] ',')* ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) """ typedargslist__32 = _argslist(Rule("tfpdef")) """ (3.2-) typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) """ varargslist__30 = _argslist(Rule("vfpdef"), old_style=True) """ (3.0, 3.1) varargslist: ((vfpdef ['=' test] ',')* ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) """ varargslist__32 = _argslist(Rule("vfpdef")) """ (3.2-) varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) """ @action(Seq(Tok("ident"), Opt(Seq(Loc(":"), Rule("test"))))) def tfpdef(self, ident_tok, annotation_opt): """(3.0-) tfpdef: NAME [':' test]""" if annotation_opt: colon_loc, annotation = annotation_opt return self._arg(ident_tok, colon_loc, annotation) return self._arg(ident_tok) vfpdef = fpdef_1 """(3.0-) vfpdef: NAME""" @action(List(Rule("fpdef"), ",", trailing=True)) def fplist(self, elts): """fplist: fpdef (',' fpdef)* [',']""" return ast.Tuple(elts=elts, ctx=None, loc=None) stmt = Alt(Rule("simple_stmt"), Rule("compound_stmt")) """stmt: simple_stmt | compound_stmt""" simple_stmt = SeqN(0, List(Rule("small_stmt"), ";", trailing=True), Tok("newline")) """simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE""" small_stmt = Alt(Rule("expr_stmt"), Rule("print_stmt"), Rule("del_stmt"), Rule("pass_stmt"), Rule("flow_stmt"), Rule("import_stmt"), Rule("global_stmt"), Rule("nonlocal_stmt"), Rule("exec_stmt"), Rule("assert_stmt")) """ (2.6, 2.7) small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt) (3.0-) small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) """ expr_stmt_1__26 = Rule("testlist") expr_stmt_1__32 = Rule("testlist_star_expr") @action(Seq(Rule("augassign"), Alt(Rule("yield_expr"), Rule("testlist")))) def expr_stmt_2(self, augassign, rhs_expr): return ast.AugAssign(op=augassign, value=rhs_expr) @action(Star(Seq(Loc("="), Alt(Rule("yield_expr"), Rule("expr_stmt_1"))))) def expr_stmt_3(self, seq): if len(seq) > 0: return ast.Assign(targets=list(map(lambda x: x[1], seq[:-1])), value=seq[-1][1], op_locs=list(map(lambda x: x[0], seq))) else: return None @action(Seq(Rule("expr_stmt_1"), Alt(expr_stmt_2, expr_stmt_3))) def expr_stmt(self, lhs, rhs): """ (2.6, 2.7, 3.0, 3.1) expr_stmt: testlist (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist))*) (3.2-) expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) """ if isinstance(rhs, ast.AugAssign): if isinstance(lhs, ast.Tuple) or isinstance(lhs, ast.List): error = diagnostic.Diagnostic( "fatal", "illegal expression for augmented assignment", {}, rhs.op.loc, [lhs.loc]) self.diagnostic_engine.process(error) else: rhs.target = self._assignable(lhs) rhs.loc = rhs.target.loc.join(rhs.value.loc) return rhs elif rhs is not None: rhs.targets = list(map(self._assignable, [lhs] + rhs.targets)) rhs.loc = lhs.loc.join(rhs.value.loc) return rhs else: return ast.Expr(value=lhs, loc=lhs.loc) testlist_star_expr = action( List(Alt(Rule("test"), Rule("star_expr")), ",", trailing=True)) \ (_wrap_tuple) """(3.2-) testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']""" augassign = Alt(Oper(ast.Add, "+="), Oper(ast.Sub, "-="), Oper(ast.MatMult, "@="), Oper(ast.Mult, "*="), Oper(ast.Div, "/="), Oper(ast.Mod, "%="), Oper(ast.BitAnd, "&="), Oper(ast.BitOr, "|="), Oper(ast.BitXor, "^="), Oper(ast.LShift, "<<="), Oper(ast.RShift, ">>="), Oper(ast.Pow, "**="), Oper(ast.FloorDiv, "//=")) """augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=')""" @action(List(Rule("test"), ",", trailing=True)) def print_stmt_1(self, values): nl, loc = True, values[-1].loc if values.trailing_comma: nl, loc = False, values.trailing_comma.loc return ast.Print(dest=None, values=values, nl=nl, dest_loc=None, loc=loc) @action(Seq(Loc(">>"), Rule("test"), Tok(","), List(Rule("test"), ",", trailing=True))) def print_stmt_2(self, dest_loc, dest, comma_tok, values): nl, loc = True, values[-1].loc if values.trailing_comma: nl, loc = False, values.trailing_comma.loc return ast.Print(dest=dest, values=values, nl=nl, dest_loc=dest_loc, loc=loc) @action(Eps()) def print_stmt_3(self, eps): return ast.Print(dest=None, values=[], nl=True, dest_loc=None, loc=None) @action(Seq(Loc("print"), Alt(print_stmt_1, print_stmt_2, print_stmt_3))) def print_stmt(self, print_loc, stmt): """ (2.6-2.7) print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] ) """ stmt.keyword_loc = print_loc if stmt.loc is None: stmt.loc = print_loc else: stmt.loc = print_loc.join(stmt.loc) return stmt @action(Seq(Loc("del"), List(Rule("expr"), ",", trailing=True))) def del_stmt(self, stmt_loc, exprs): # Python uses exprlist here, but does *not* obey the usual # tuple-wrapping semantics, so we embed the rule directly. """del_stmt: 'del' exprlist""" return ast.Delete(targets=[self._assignable(expr, is_delete=True) for expr in exprs], loc=stmt_loc.join(exprs[-1].loc), keyword_loc=stmt_loc) @action(Loc("pass")) def pass_stmt(self, stmt_loc): """pass_stmt: 'pass'""" return ast.Pass(loc=stmt_loc, keyword_loc=stmt_loc) flow_stmt = Alt(Rule("break_stmt"), Rule("continue_stmt"), Rule("return_stmt"), Rule("raise_stmt"), Rule("yield_stmt")) """flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt""" @action(Loc("break")) def break_stmt(self, stmt_loc): """break_stmt: 'break'""" return ast.Break(loc=stmt_loc, keyword_loc=stmt_loc) @action(Loc("continue")) def continue_stmt(self, stmt_loc): """continue_stmt: 'continue'""" return ast.Continue(loc=stmt_loc, keyword_loc=stmt_loc) @action(Seq(Loc("return"), Opt(Rule("testlist")))) def return_stmt(self, stmt_loc, values): """return_stmt: 'return' [testlist]""" loc = stmt_loc if values: loc = loc.join(values.loc) return ast.Return(value=values, loc=loc, keyword_loc=stmt_loc) @action(Rule("yield_expr")) def yield_stmt(self, expr): """yield_stmt: yield_expr""" return ast.Expr(value=expr, loc=expr.loc) @action(Seq(Loc("raise"), Opt(Seq(Rule("test"), Opt(Seq(Tok(","), Rule("test"), Opt(SeqN(1, Tok(","), Rule("test"))))))))) def raise_stmt__26(self, raise_loc, type_opt): """(2.6, 2.7) raise_stmt: 'raise' [test [',' test [',' test]]]""" type_ = inst = tback = None loc = raise_loc if type_opt: type_, inst_opt = type_opt loc = loc.join(type_.loc) if inst_opt: _, inst, tback = inst_opt loc = loc.join(inst.loc) if tback: loc = loc.join(tback.loc) return ast.Raise(exc=type_, inst=inst, tback=tback, cause=None, keyword_loc=raise_loc, from_loc=None, loc=loc) @action(Seq(Loc("raise"), Opt(Seq(Rule("test"), Opt(Seq(Loc("from"), Rule("test"))))))) def raise_stmt__30(self, raise_loc, exc_opt): """(3.0-) raise_stmt: 'raise' [test ['from' test]]""" exc = from_loc = cause = None loc = raise_loc if exc_opt: exc, cause_opt = exc_opt loc = loc.join(exc.loc) if cause_opt: from_loc, cause = cause_opt loc = loc.join(cause.loc) return ast.Raise(exc=exc, inst=None, tback=None, cause=cause, keyword_loc=raise_loc, from_loc=from_loc, loc=loc) import_stmt = Alt(Rule("import_name"), Rule("import_from")) """import_stmt: import_name | import_from""" @action(Seq(Loc("import"), Rule("dotted_as_names"))) def import_name(self, import_loc, names): """import_name: 'import' dotted_as_names""" return ast.Import(names=names, keyword_loc=import_loc, loc=import_loc.join(names[-1].loc)) @action(Loc(".")) def import_from_1(self, loc): return 1, loc @action(Loc("...")) def import_from_2(self, loc): return 3, loc @action(Seq(Star(Alt(import_from_1, import_from_2)), Rule("dotted_name"))) def import_from_3(self, dots, dotted_name): dots_loc, dots_count = None, 0 if any(dots): dots_loc = dots[0][1].join(dots[-1][1]) dots_count = sum([count for count, loc in dots]) return (dots_loc, dots_count), dotted_name @action(Plus(Alt(import_from_1, import_from_2))) def import_from_4(self, dots): dots_loc = dots[0][1].join(dots[-1][1]) dots_count = sum([count for count, loc in dots]) return (dots_loc, dots_count), None @action(Loc("*")) def import_from_5(self, star_loc): return (None, 0), \ [ast.alias(name="*", asname=None, name_loc=star_loc, as_loc=None, asname_loc=None, loc=star_loc)], \ None @action(Rule("atom_5")) def import_from_7(self, string): return (None, 0), (string.loc, string.s) @action(Rule("import_as_names")) def import_from_6(self, names): return (None, 0), names, None @action(Seq(Loc("from"), Alt(import_from_3, import_from_4, import_from_7), Loc("import"), Alt(import_from_5, Seq(Loc("("), Rule("import_as_names"), Loc(")")), import_from_6))) def import_from(self, from_loc, module_name, import_loc, names): """ (2.6, 2.7) import_from: ('from' ('.'* dotted_name | '.'+) 'import' ('*' | '(' import_as_names ')' | import_as_names)) (3.0-) # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 'import' ('*' | '(' import_as_names ')' | import_as_names)) """ (dots_loc, dots_count), dotted_name_opt = module_name module_loc = module = None if dotted_name_opt: module_loc, module = dotted_name_opt lparen_loc, names, rparen_loc = names loc = from_loc.join(names[-1].loc) if rparen_loc: loc = loc.join(rparen_loc) if module == "__future__": self.add_flags([x.name for x in names]) return ast.ImportFrom(names=names, module=module, level=dots_count, keyword_loc=from_loc, dots_loc=dots_loc, module_loc=module_loc, import_loc=import_loc, lparen_loc=lparen_loc, rparen_loc=rparen_loc, loc=loc) @action(Seq(Tok("ident"), Opt(Seq(Loc("as"), Tok("ident"))))) def import_as_name(self, name_tok, as_name_opt): """import_as_name: NAME ['as' NAME]""" asname_name = asname_loc = as_loc = None loc = name_tok.loc if as_name_opt: as_loc, asname = as_name_opt asname_name = asname.value asname_loc = asname.loc loc = loc.join(asname.loc) return ast.alias(name=name_tok.value, asname=asname_name, loc=loc, name_loc=name_tok.loc, as_loc=as_loc, asname_loc=asname_loc) @action(Seq(Rule("dotted_name"), Opt(Seq(Loc("as"), Tok("ident"))))) def dotted_as_name(self, dotted_name, as_name_opt): """dotted_as_name: dotted_name ['as' NAME]""" asname_name = asname_loc = as_loc = None dotted_name_loc, dotted_name_name = dotted_name loc = dotted_name_loc if as_name_opt: as_loc, asname = as_name_opt asname_name = asname.value asname_loc = asname.loc loc = loc.join(asname.loc) return ast.alias(name=dotted_name_name, asname=asname_name, loc=loc, name_loc=dotted_name_loc, as_loc=as_loc, asname_loc=asname_loc) @action(Seq(Rule("atom_5"), Opt(Seq(Loc("as"), Tok("ident"))))) def str_as_name(self, string, as_name_opt): asname_name = asname_loc = as_loc = None loc = string.loc if as_name_opt: as_loc, asname = as_name_opt asname_name = asname.value asname_loc = asname.loc loc = loc.join(asname.loc) return ast.alias(name=string.s, asname=asname_name, loc=loc, name_loc=string.loc, as_loc=as_loc, asname_loc=asname_loc) import_as_names = List(Rule("import_as_name"), ",", trailing=True) """import_as_names: import_as_name (',' import_as_name)* [',']""" dotted_as_names = List(Alt(Rule("dotted_as_name"), Rule("str_as_name")), ",", trailing=False) """dotted_as_names: dotted_as_name (',' dotted_as_name)*""" @action(List(Tok("ident"), ".", trailing=False)) def dotted_name(self, idents): """dotted_name: NAME ('.' NAME)*""" return idents[0].loc.join(idents[-1].loc), \ ".".join(list(map(lambda x: x.value, idents))) @action(Seq(Loc("global"), List(Tok("ident"), ",", trailing=False))) def global_stmt(self, global_loc, names): """global_stmt: 'global' NAME (',' NAME)*""" return ast.Global(names=list(map(lambda x: x.value, names)), name_locs=list(map(lambda x: x.loc, names)), keyword_loc=global_loc, loc=global_loc.join(names[-1].loc)) @action(Seq(Loc("exec"), Rule("expr"), Opt(Seq(Loc("in"), Rule("test"), Opt(SeqN(1, Loc(","), Rule("test"))))))) def exec_stmt(self, exec_loc, body, in_opt): """(2.6, 2.7) exec_stmt: 'exec' expr ['in' test [',' test]]""" in_loc, globals, locals = None, None, None loc = exec_loc.join(body.loc) if in_opt: in_loc, globals, locals = in_opt if locals: loc = loc.join(locals.loc) else: loc = loc.join(globals.loc) return ast.Exec(body=body, locals=locals, globals=globals, loc=loc, keyword_loc=exec_loc, in_loc=in_loc) @action(Seq(Loc("nonlocal"), List(Tok("ident"), ",", trailing=False))) def nonlocal_stmt(self, nonlocal_loc, names): """(3.0-) nonlocal_stmt: 'nonlocal' NAME (',' NAME)*""" return ast.Nonlocal(names=list(map(lambda x: x.value, names)), name_locs=list(map(lambda x: x.loc, names)), keyword_loc=nonlocal_loc, loc=nonlocal_loc.join(names[-1].loc)) @action(Seq(Loc("assert"), Rule("test"), Opt(SeqN(1, Tok(","), Rule("test"))))) def assert_stmt(self, assert_loc, test, msg): """assert_stmt: 'assert' test [',' test]""" loc = assert_loc.join(test.loc) if msg: loc = loc.join(msg.loc) return ast.Assert(test=test, msg=msg, loc=loc, keyword_loc=assert_loc) @action(Alt(Rule("if_stmt"), Rule("while_stmt"), Rule("for_stmt"), Rule("try_stmt"), Rule("with_stmt"), Rule("funcdef"), Rule("classdef"), Rule("decorated"))) def compound_stmt(self, stmt): """compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated""" return [stmt] @action(Seq(Loc("if"), Rule("test"), Loc(":"), Rule("suite"), Star(Seq(Loc("elif"), Rule("test"), Loc(":"), Rule("suite"))), Opt(Seq(Loc("else"), Loc(":"), Rule("suite"))))) def if_stmt(self, if_loc, test, if_colon_loc, body, elifs, else_opt): """if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]""" stmt = ast.If(orelse=[], else_loc=None, else_colon_loc=None) if else_opt: stmt.else_loc, stmt.else_colon_loc, stmt.orelse = else_opt for elif_ in reversed(elifs): stmt.keyword_loc, stmt.test, stmt.if_colon_loc, stmt.body = elif_ stmt.loc = stmt.keyword_loc.join(stmt.body[-1].loc) if stmt.orelse: stmt.loc = stmt.loc.join(stmt.orelse[-1].loc) stmt = ast.If(orelse=[stmt], else_loc=None, else_colon_loc=None) stmt.keyword_loc, stmt.test, stmt.if_colon_loc, stmt.body = \ if_loc, test, if_colon_loc, body stmt.loc = stmt.keyword_loc.join(stmt.body[-1].loc) if stmt.orelse: stmt.loc = stmt.loc.join(stmt.orelse[-1].loc) return stmt @action(Seq(Loc("while"), Rule("test"), Loc(":"), Rule("suite"), Opt(Seq(Loc("else"), Loc(":"), Rule("suite"))))) def while_stmt(self, while_loc, test, while_colon_loc, body, else_opt): """while_stmt: 'while' test ':' suite ['else' ':' suite]""" stmt = ast.While(test=test, body=body, orelse=[], keyword_loc=while_loc, while_colon_loc=while_colon_loc, else_loc=None, else_colon_loc=None, loc=while_loc.join(body[-1].loc)) if else_opt: stmt.else_loc, stmt.else_colon_loc, stmt.orelse = else_opt stmt.loc = stmt.loc.join(stmt.orelse[-1].loc) return stmt @action(Seq(Loc("for"), Rule("exprlist"), Loc("in"), Rule("testlist"), Loc(":"), Rule("suite"), Opt(Seq(Loc("else"), Loc(":"), Rule("suite"))))) def for_stmt(self, for_loc, target, in_loc, iter, for_colon_loc, body, else_opt): """for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]""" stmt = ast.For(target=self._assignable(target), iter=iter, body=body, orelse=[], keyword_loc=for_loc, in_loc=in_loc, for_colon_loc=for_colon_loc, else_loc=None, else_colon_loc=None, loc=for_loc.join(body[-1].loc)) if else_opt: stmt.else_loc, stmt.else_colon_loc, stmt.orelse = else_opt stmt.loc = stmt.loc.join(stmt.orelse[-1].loc) return stmt @action(Seq(Plus(Seq(Rule("except_clause"), Loc(":"), Rule("suite"))), Opt(Seq(Loc("else"), Loc(":"), Rule("suite"))), Opt(Seq(Loc("finally"), Loc(":"), Rule("suite"))))) def try_stmt_1(self, clauses, else_opt, finally_opt): handlers = [] for clause in clauses: handler, handler.colon_loc, handler.body = clause handler.loc = handler.loc.join(handler.body[-1].loc) handlers.append(handler) else_loc, else_colon_loc, orelse = None, None, [] loc = handlers[-1].loc if else_opt: else_loc, else_colon_loc, orelse = else_opt loc = orelse[-1].loc finally_loc, finally_colon_loc, finalbody = None, None, [] if finally_opt: finally_loc, finally_colon_loc, finalbody = finally_opt loc = finalbody[-1].loc stmt = ast.Try(body=None, handlers=handlers, orelse=orelse, finalbody=finalbody, else_loc=else_loc, else_colon_loc=else_colon_loc, finally_loc=finally_loc, finally_colon_loc=finally_colon_loc, loc=loc) return stmt @action(Seq(Loc("finally"), Loc(":"), Rule("suite"))) def try_stmt_2(self, finally_loc, finally_colon_loc, finalbody): return ast.Try(body=None, handlers=[], orelse=[], finalbody=finalbody, else_loc=None, else_colon_loc=None, finally_loc=finally_loc, finally_colon_loc=finally_colon_loc, loc=finalbody[-1].loc) @action(Seq(Loc("try"), Loc(":"), Rule("suite"), Alt(try_stmt_1, try_stmt_2))) def try_stmt(self, try_loc, try_colon_loc, body, stmt): """ try_stmt: ('try' ':' suite ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] | 'finally' ':' suite)) """ stmt.keyword_loc, stmt.try_colon_loc, stmt.body = \ try_loc, try_colon_loc, body stmt.loc = stmt.loc.join(try_loc) return stmt @action(Seq(Loc("with"), Rule("test"), Opt(Rule("with_var")), Loc(":"), Rule("suite"))) def with_stmt__26(self, with_loc, context, with_var, colon_loc, body): """(2.6, 3.0) with_stmt: 'with' test [ with_var ] ':' suite""" if with_var: as_loc, optional_vars = with_var item = ast.withitem(context_expr=context, optional_vars=optional_vars, as_loc=as_loc, loc=context.loc.join(optional_vars.loc)) else: item = ast.withitem(context_expr=context, optional_vars=None, as_loc=None, loc=context.loc) return ast.With(items=[item], body=body, keyword_loc=with_loc, colon_loc=colon_loc, loc=with_loc.join(body[-1].loc)) with_var = Seq(Loc("as"), Rule("expr")) """(2.6, 3.0) with_var: 'as' expr""" @action(Seq(Loc("with"), List(Rule("with_item"), ",", trailing=False), Loc(":"), Rule("suite"))) def with_stmt__27(self, with_loc, items, colon_loc, body): """(2.7, 3.1-) with_stmt: 'with' with_item (',' with_item)* ':' suite""" return ast.With(items=items, body=body, keyword_loc=with_loc, colon_loc=colon_loc, loc=with_loc.join(body[-1].loc)) @action(Seq(Rule("test"), Opt(Seq(Loc("as"), Rule("expr"))))) def with_item(self, context, as_opt): """(2.7, 3.1-) with_item: test ['as' expr]""" if as_opt: as_loc, optional_vars = as_opt return ast.withitem(context_expr=context, optional_vars=optional_vars, as_loc=as_loc, loc=context.loc.join(optional_vars.loc)) else: return ast.withitem(context_expr=context, optional_vars=None, as_loc=None, loc=context.loc) @action(Seq(Alt(Loc("as"), Loc(",")), Rule("test"))) def except_clause_1__26(self, as_loc, name): return as_loc, None, name @action(Seq(Loc("as"), Tok("ident"))) def except_clause_1__30(self, as_loc, name): return as_loc, name, None @action(Seq(Loc("except"), Opt(Seq(Rule("test"), Opt(Rule("except_clause_1")))))) def except_clause(self, except_loc, exc_opt): """ (2.6, 2.7) except_clause: 'except' [test [('as' | ',') test]] (3.0-) except_clause: 'except' [test ['as' NAME]] """ type_ = name = as_loc = name_loc = None loc = except_loc if exc_opt: type_, name_opt = exc_opt loc = loc.join(type_.loc) if name_opt: as_loc, name_tok, name_node = name_opt if name_tok: name = name_tok.value name_loc = name_tok.loc else: name = name_node name_loc = name_node.loc loc = loc.join(name_loc) return ast.ExceptHandler(type=type_, name=name, except_loc=except_loc, as_loc=as_loc, name_loc=name_loc, loc=loc) @action(Plus(Rule("stmt"))) def suite_1(self, stmts): return reduce(list.__add__, stmts, []) suite = Alt(Rule("simple_stmt"), SeqN(2, Tok("newline"), Tok("indent"), suite_1, Tok("dedent"))) """suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT""" # 2.x-only backwards compatibility start testlist_safe = action(List(Rule("old_test"), ",", trailing=False))(_wrap_tuple) """(2.6, 2.7) testlist_safe: old_test [(',' old_test)+ [',']]""" old_test = Alt(Rule("or_test"), Rule("old_lambdef")) """(2.6, 2.7) old_test: or_test | old_lambdef""" @action(Seq(Loc("lambda"), Opt(Rule("varargslist")), Loc(":"), Rule("old_test"))) def old_lambdef(self, lambda_loc, args_opt, colon_loc, body): """(2.6, 2.7) old_lambdef: 'lambda' [varargslist] ':' old_test""" if args_opt is None: args_opt = self._arguments() args_opt.loc = colon_loc.begin() return ast.Lambda(args=args_opt, body=body, lambda_loc=lambda_loc, colon_loc=colon_loc, loc=lambda_loc.join(body.loc)) # 2.x-only backwards compatibility end @action(Seq(Rule("or_test"), Opt(Seq(Loc("if"), Rule("or_test"), Loc("else"), Rule("test"))))) def test_1(self, lhs, rhs_opt): if rhs_opt is not None: if_loc, test, else_loc, orelse = rhs_opt return ast.IfExp(test=test, body=lhs, orelse=orelse, if_loc=if_loc, else_loc=else_loc, loc=lhs.loc.join(orelse.loc)) return lhs test = Alt(test_1, Rule("lambdef")) """test: or_test ['if' or_test 'else' test] | lambdef""" test_nocond = Alt(Rule("or_test"), Rule("lambdef_nocond")) """(3.0-) test_nocond: or_test | lambdef_nocond""" def lambdef_action(self, lambda_loc, args_opt, colon_loc, body): if args_opt is None: args_opt = self._arguments() args_opt.loc = colon_loc.begin() return ast.Lambda(args=args_opt, body=body, lambda_loc=lambda_loc, colon_loc=colon_loc, loc=lambda_loc.join(body.loc)) lambdef = action( Seq(Loc("lambda"), Opt(Rule("varargslist")), Loc(":"), Rule("test"))) \ (lambdef_action) """lambdef: 'lambda' [varargslist] ':' test""" lambdef_nocond = action( Seq(Loc("lambda"), Opt(Rule("varargslist")), Loc(":"), Rule("test_nocond"))) \ (lambdef_action) """(3.0-) lambdef_nocond: 'lambda' [varargslist] ':' test_nocond""" @action(Seq(Rule("and_test"), Star(Seq(Loc("or"), Rule("and_test"))))) def or_test(self, lhs, rhs): """or_test: and_test ('or' and_test)*""" if len(rhs) > 0: return ast.BoolOp(op=ast.Or(), values=[lhs] + list(map(lambda x: x[1], rhs)), loc=lhs.loc.join(rhs[-1][1].loc), op_locs=list(map(lambda x: x[0], rhs))) else: return lhs @action(Seq(Rule("not_test"), Star(Seq(Loc("and"), Rule("not_test"))))) def and_test(self, lhs, rhs): """and_test: not_test ('and' not_test)*""" if len(rhs) > 0: return ast.BoolOp(op=ast.And(), values=[lhs] + list(map(lambda x: x[1], rhs)), loc=lhs.loc.join(rhs[-1][1].loc), op_locs=list(map(lambda x: x[0], rhs))) else: return lhs @action(Seq(Oper(ast.Not, "not"), Rule("not_test"))) def not_test_1(self, op, operand): return ast.UnaryOp(op=op, operand=operand, loc=op.loc.join(operand.loc)) not_test = Alt(not_test_1, Rule("comparison")) """not_test: 'not' not_test | comparison""" comparison_1__26 = Seq(Rule("expr"), Star(Seq(Rule("comp_op"), Rule("expr")))) comparison_1__30 = Seq(Rule("star_expr"), Star(Seq(Rule("comp_op"), Rule("star_expr")))) comparison_1__32 = comparison_1__26 @action(Rule("comparison_1")) def comparison(self, lhs, rhs): """ (2.6, 2.7) comparison: expr (comp_op expr)* (3.0, 3.1) comparison: star_expr (comp_op star_expr)* (3.2-) comparison: expr (comp_op expr)* """ if len(rhs) > 0: return ast.Compare(left=lhs, ops=list(map(lambda x: x[0], rhs)), comparators=list(map(lambda x: x[1], rhs)), loc=lhs.loc.join(rhs[-1][1].loc)) else: return lhs @action(Seq(Opt(Loc("*")), Rule("expr"))) def star_expr__30(self, star_opt, expr): """(3.0, 3.1) star_expr: ['*'] expr""" if star_opt: return ast.Starred(value=expr, ctx=None, star_loc=star_opt, loc=expr.loc.join(star_opt)) return expr @action(Seq(Loc("*"), Rule("expr"))) def star_expr__32(self, star_loc, expr): """(3.0-) star_expr: '*' expr""" return ast.Starred(value=expr, ctx=None, star_loc=star_loc, loc=expr.loc.join(star_loc)) comp_op = Alt(Oper(ast.Lt, "<"), Oper(ast.Gt, ">"), Oper(ast.Eq, "=="), Oper(ast.GtE, ">="), Oper(ast.LtE, "<="), Oper(ast.NotEq, "<>"), Oper(ast.NotEq, "!="), Oper(ast.In, "in"), Oper(ast.NotIn, "not", "in"), Oper(ast.IsNot, "is", "not"), Oper(ast.Is, "is")) """ (2.6, 2.7) comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' (3.0-) comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not' """ expr = BinOper("xor_expr", Oper(ast.BitOr, "|")) """expr: xor_expr ('|' xor_expr)*""" xor_expr = BinOper("and_expr", Oper(ast.BitXor, "^")) """xor_expr: and_expr ('^' and_expr)*""" and_expr = BinOper("shift_expr", Oper(ast.BitAnd, "&")) """and_expr: shift_expr ('&' shift_expr)*""" shift_expr = BinOper("arith_expr", Alt(Oper(ast.LShift, "<<"), Oper(ast.RShift, ">>"))) """shift_expr: arith_expr (('<<'|'>>') arith_expr)*""" arith_expr = BinOper("term", Alt(Oper(ast.Add, "+"), Oper(ast.Sub, "-"))) """arith_expr: term (('+'|'-') term)*""" term = BinOper("factor", Alt(Oper(ast.Mult, "*"), Oper(ast.MatMult, "@"), Oper(ast.Div, "/"), Oper(ast.Mod, "%"), Oper(ast.FloorDiv, "//"))) """term: factor (('*'|'/'|'%'|'//') factor)*""" @action(Seq(Alt(Oper(ast.UAdd, "+"), Oper(ast.USub, "-"), Oper(ast.Invert, "~")), Rule("factor"))) def factor_1(self, op, factor): return ast.UnaryOp(op=op, operand=factor, loc=op.loc.join(factor.loc)) factor = Alt(factor_1, Rule("power")) """factor: ('+'|'-'|'~') factor | power""" @action(Seq(Rule("atom"), Star(Rule("trailer")), Opt(Seq(Loc("**"), Rule("factor"))))) def power(self, atom, trailers, factor_opt): """power: atom trailer* ['**' factor]""" for trailer in trailers: if isinstance(trailer, ast.Attribute) or isinstance(trailer, ast.Subscript): trailer.value = atom elif isinstance(trailer, ast.Call): trailer.func = atom trailer.loc = atom.loc.join(trailer.loc) atom = trailer if factor_opt: op_loc, factor = factor_opt return ast.BinOp(left=atom, op=ast.Pow(loc=op_loc), right=factor, loc=atom.loc.join(factor.loc)) return atom @action(Rule("testlist1")) def atom_1(self, expr): return ast.Repr(value=expr, loc=None) @action(Tok("ident")) def atom_2(self, tok): return ast.Name(id=tok.value, loc=tok.loc, ctx=None) @action(Alt(Tok("int"), Tok("float"), Tok("complex"))) def atom_3(self, tok): return ast.Num(n=tok.value, loc=tok.loc) @action(Seq(Tok("strbegin"), Tok("strdata"), Tok("strend"))) def atom_4(self, begin_tok, data_tok, end_tok): return ast.Str(s=data_tok.value, begin_loc=begin_tok.loc, end_loc=end_tok.loc, loc=begin_tok.loc.join(end_tok.loc)) @action(Plus(atom_4)) def atom_5(self, strings): joint = "" if all(isinstance(x.s, bytes) for x in strings): joint = b"" return ast.Str(s=joint.join([x.s for x in strings]), begin_loc=strings[0].begin_loc, end_loc=strings[-1].end_loc, loc=strings[0].loc.join(strings[-1].loc)) atom_6__26 = Rule("dictmaker") atom_6__27 = Rule("dictorsetmaker") atom__26 = Alt(BeginEnd("(", Opt(Alt(Rule("yield_expr"), Rule("testlist_comp"))), ")", empty=lambda self: ast.Tuple(elts=[], ctx=None, loc=None)), BeginEnd("[", Opt(Rule("listmaker")), "]", empty=lambda self: ast.List(elts=[], ctx=None, loc=None)), BeginEnd("{", Opt(Rule("atom_6")), "}", empty=lambda self: ast.Dict(keys=[], values=[], colon_locs=[], loc=None)), BeginEnd("`", atom_1, "`"), atom_2, atom_3, atom_5) """ (2.6) atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+) (2.7) atom: ('(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dictorsetmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+) """ @action(Loc("...")) def atom_7(self, loc): return ast.Ellipsis(loc=loc) @action(Alt(Tok("None"), Tok("True"), Tok("False"))) def atom_8(self, tok): if tok.kind == "None": value = None elif tok.kind == "True": value = True elif tok.kind == "False": value = False return ast.NameConstant(value=value, loc=tok.loc) atom__30 = Alt(BeginEnd("(", Opt(Alt(Rule("yield_expr"), Rule("testlist_comp"))), ")", empty=lambda self: ast.Tuple(elts=[], ctx=None, loc=None)), BeginEnd("[", Opt(Rule("testlist_comp__list")), "]", empty=lambda self: ast.List(elts=[], ctx=None, loc=None)), BeginEnd("{", Opt(Rule("dictorsetmaker")), "}", empty=lambda self: ast.Dict(keys=[], values=[], colon_locs=[], loc=None)), atom_2, atom_3, atom_5, atom_7, atom_8) """ (3.0-) atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') """ def list_gen_action(self, lhs, rhs): if rhs is None: # (x) return lhs elif isinstance(rhs, ast.Tuple) or isinstance(rhs, ast.List): rhs.elts = [lhs] + rhs.elts return rhs elif isinstance(rhs, ast.ListComp) or isinstance(rhs, ast.GeneratorExp): rhs.elt = lhs return rhs @action(Rule("list_for")) def listmaker_1(self, compose): return ast.ListComp(generators=compose([]), loc=None) @action(List(Rule("test"), ",", trailing=True, leading=False)) def listmaker_2(self, elts): return ast.List(elts=elts, ctx=None, loc=None) listmaker = action( Seq(Rule("test"), Alt(listmaker_1, listmaker_2))) \ (list_gen_action) """listmaker: test ( list_for | (',' test)* [','] )""" testlist_comp_1__26 = Rule("test") testlist_comp_1__32 = Alt(Rule("test"), Rule("star_expr")) @action(Rule("comp_for")) def testlist_comp_2(self, compose): return ast.GeneratorExp(generators=compose([]), loc=None) @action(List(Rule("testlist_comp_1"), ",", trailing=True, leading=False)) def testlist_comp_3(self, elts): if elts == [] and not elts.trailing_comma: return None else: return ast.Tuple(elts=elts, ctx=None, loc=None) testlist_comp = action( Seq(Rule("testlist_comp_1"), Alt(testlist_comp_2, testlist_comp_3))) \ (list_gen_action) """ (2.6) testlist_gexp: test ( gen_for | (',' test)* [','] ) (2.7, 3.0, 3.1) testlist_comp: test ( comp_for | (',' test)* [','] ) (3.2-) testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) """ @action(Rule("comp_for")) def testlist_comp__list_1(self, compose): return ast.ListComp(generators=compose([]), loc=None) @action(List(Rule("testlist_comp_1"), ",", trailing=True, leading=False)) def testlist_comp__list_2(self, elts): return ast.List(elts=elts, ctx=None, loc=None) testlist_comp__list = action( Seq(Rule("testlist_comp_1"), Alt(testlist_comp__list_1, testlist_comp__list_2))) \ (list_gen_action) """Same grammar as testlist_comp, but different semantic action.""" @action(Seq(Loc("."), Tok("ident"))) def trailer_1(self, dot_loc, ident_tok): return ast.Attribute(attr=ident_tok.value, ctx=None, loc=dot_loc.join(ident_tok.loc), attr_loc=ident_tok.loc, dot_loc=dot_loc) trailer = Alt(BeginEnd("(", Opt(Rule("arglist")), ")", empty=_empty_arglist), BeginEnd("[", Rule("subscriptlist"), "]"), trailer_1) """trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME""" @action(List(Rule("subscript"), ",", trailing=True)) def subscriptlist(self, subscripts): """subscriptlist: subscript (',' subscript)* [',']""" if len(subscripts) == 1: return ast.Subscript(slice=subscripts[0], ctx=None, loc=None) elif all([isinstance(x, ast.Index) for x in subscripts]): elts = [x.value for x in subscripts] loc = subscripts[0].loc.join(subscripts[-1].loc) index = ast.Index(value=ast.Tuple(elts=elts, ctx=None, begin_loc=None, end_loc=None, loc=loc), loc=loc) return ast.Subscript(slice=index, ctx=None, loc=None) else: extslice = ast.ExtSlice(dims=subscripts, loc=subscripts[0].loc.join(subscripts[-1].loc)) return ast.Subscript(slice=extslice, ctx=None, loc=None) @action(Seq(Loc("."), Loc("."), Loc("."))) def subscript_1(self, dot_1_loc, dot_2_loc, dot_3_loc): return ast.Ellipsis(loc=dot_1_loc.join(dot_3_loc)) @action(Seq(Opt(Rule("test")), Loc(":"), Opt(Rule("test")), Opt(Rule("sliceop")))) def subscript_2(self, lower_opt, colon_loc, upper_opt, step_opt): loc = colon_loc if lower_opt: loc = loc.join(lower_opt.loc) if upper_opt: loc = loc.join(upper_opt.loc) step_colon_loc = step = None if step_opt: step_colon_loc, step = step_opt loc = loc.join(step_colon_loc) if step: loc = loc.join(step.loc) return ast.Slice(lower=lower_opt, upper=upper_opt, step=step, loc=loc, bound_colon_loc=colon_loc, step_colon_loc=step_colon_loc) @action(Rule("test")) def subscript_3(self, expr): return ast.Index(value=expr, loc=expr.loc) subscript__26 = Alt(subscript_1, subscript_2, subscript_3) """(2.6, 2.7) subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]""" subscript__30 = Alt(subscript_2, subscript_3) """(3.0-) subscript: test | [test] ':' [test] [sliceop]""" sliceop = Seq(Loc(":"), Opt(Rule("test"))) """sliceop: ':' [test]""" exprlist_1__26 = List(Rule("expr"), ",", trailing=True) exprlist_1__30 = List(Rule("star_expr"), ",", trailing=True) exprlist_1__32 = List(Alt(Rule("expr"), Rule("star_expr")), ",", trailing=True) @action(Rule("exprlist_1")) def exprlist(self, exprs): """ (2.6, 2.7) exprlist: expr (',' expr)* [','] (3.0, 3.1) exprlist: star_expr (',' star_expr)* [','] (3.2-) exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] """ return self._wrap_tuple(exprs) @action(List(Rule("test"), ",", trailing=True)) def testlist(self, exprs): """testlist: test (',' test)* [',']""" return self._wrap_tuple(exprs) @action(List(Seq(Rule("test"), Loc(":"), Rule("test")), ",", trailing=True)) def dictmaker(self, elts): """(2.6) dictmaker: test ':' test (',' test ':' test)* [',']""" return ast.Dict(keys=list(map(lambda x: x[0], elts)), values=list(map(lambda x: x[2], elts)), colon_locs=list(map(lambda x: x[1], elts)), loc=None) dictorsetmaker_1 = Seq(Rule("test"), Loc(":"), Rule("test")) @action(Seq(dictorsetmaker_1, Alt(Rule("comp_for"), List(dictorsetmaker_1, ",", leading=False, trailing=True)))) def dictorsetmaker_2(self, first, elts): if isinstance(elts, commalist): elts.insert(0, first) return ast.Dict(keys=list(map(lambda x: x[0], elts)), values=list(map(lambda x: x[2], elts)), colon_locs=list(map(lambda x: x[1], elts)), loc=None) else: return ast.DictComp(key=first[0], value=first[2], generators=elts([]), colon_loc=first[1], begin_loc=None, end_loc=None, loc=None) @action(Seq(Rule("test"), Alt(Rule("comp_for"), List(Rule("test"), ",", leading=False, trailing=True)))) def dictorsetmaker_3(self, first, elts): if isinstance(elts, commalist): elts.insert(0, first) return ast.Set(elts=elts, loc=None) else: return ast.SetComp(elt=first, generators=elts([]), begin_loc=None, end_loc=None, loc=None) dictorsetmaker = Alt(dictorsetmaker_2, dictorsetmaker_3) """ (2.7-) dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | (test (comp_for | (',' test)* [','])) ) """ @action(Seq(Loc("class"), Tok("ident"), Opt(Seq(Loc("("), List(Rule("test"), ",", trailing=True), Loc(")"))), Loc(":"), Rule("suite"))) def classdef__26(self, class_loc, name_tok, bases_opt, colon_loc, body): """(2.6, 2.7) classdef: 'class' NAME ['(' [testlist] ')'] ':' suite""" bases, lparen_loc, rparen_loc = [], None, None if bases_opt: lparen_loc, bases, rparen_loc = bases_opt return ast.ClassDef(name=name_tok.value, bases=bases, keywords=[], starargs=None, kwargs=None, body=body, decorator_list=[], at_locs=[], keyword_loc=class_loc, lparen_loc=lparen_loc, star_loc=None, dstar_loc=None, rparen_loc=rparen_loc, name_loc=name_tok.loc, colon_loc=colon_loc, loc=class_loc.join(body[-1].loc)) @action(Seq(Loc("class"), Tok("ident"), Opt(Seq(Loc("("), Rule("arglist"), Loc(")"))), Loc(":"), Rule("suite"))) def classdef__30(self, class_loc, name_tok, arglist_opt, colon_loc, body): """(3.0) classdef: 'class' NAME ['(' [testlist] ')'] ':' suite""" arglist, lparen_loc, rparen_loc = [], None, None bases, keywords, starargs, kwargs = [], [], None, None star_loc, dstar_loc = None, None if arglist_opt: lparen_loc, arglist, rparen_loc = arglist_opt bases, keywords, starargs, kwargs = \ arglist.args, arglist.keywords, arglist.starargs, arglist.kwargs star_loc, dstar_loc = arglist.star_loc, arglist.dstar_loc return ast.ClassDef(name=name_tok.value, bases=bases, keywords=keywords, starargs=starargs, kwargs=kwargs, body=body, decorator_list=[], at_locs=[], keyword_loc=class_loc, lparen_loc=lparen_loc, star_loc=star_loc, dstar_loc=dstar_loc, rparen_loc=rparen_loc, name_loc=name_tok.loc, colon_loc=colon_loc, loc=class_loc.join(body[-1].loc)) @action(Seq(Loc("*"), Rule("test"), Star(SeqN(1, Tok(","), Rule("argument"))), Opt(Seq(Tok(","), Loc("**"), Rule("test"))))) def arglist_1(self, star_loc, stararg, postargs, kwarg_opt): dstar_loc = kwarg = None if kwarg_opt: _, dstar_loc, kwarg = kwarg_opt for postarg in postargs: if not isinstance(postarg, ast.keyword): error = diagnostic.Diagnostic( "fatal", "only named arguments may follow *expression", {}, postarg.loc, [star_loc.join(stararg.loc)]) self.diagnostic_engine.process(error) return postargs, \ ast.Call(args=[], keywords=[], starargs=stararg, kwargs=kwarg, star_loc=star_loc, dstar_loc=dstar_loc, loc=None) @action(Seq(Loc("**"), Rule("test"))) def arglist_2(self, dstar_loc, kwarg): return [], \ ast.Call(args=[], keywords=[], starargs=None, kwargs=kwarg, star_loc=None, dstar_loc=dstar_loc, loc=None) @action(Seq(Rule("argument"), Alt(SeqN(1, Tok(","), Alt(Rule("arglist_1"), Rule("arglist_2"), Rule("arglist_3"), Eps())), Eps()))) def arglist_3(self, arg, cont): if cont is None: return [arg], self._empty_arglist() else: args, rest = cont return [arg] + args, rest @action(Alt(Rule("arglist_1"), Rule("arglist_2"), Rule("arglist_3"))) def arglist(self, args, call): """arglist: (argument ',')* (argument [','] | '*' test (',' argument)* [',' '**' test] | '**' test)""" for arg in args: if isinstance(arg, ast.keyword): call.keywords.append(arg) elif len(call.keywords) > 0: error = diagnostic.Diagnostic( "fatal", "non-keyword arg after keyword arg", {}, arg.loc, [call.keywords[-1].loc]) self.diagnostic_engine.process(error) else: call.args.append(arg) return call @action(Seq(Loc("="), Rule("test"))) def argument_1(self, equals_loc, rhs): def thunk(lhs): if not isinstance(lhs, ast.Name): error = diagnostic.Diagnostic( "fatal", "keyword must be an identifier", {}, lhs.loc) self.diagnostic_engine.process(error) return ast.keyword(arg=lhs.id, value=rhs, loc=lhs.loc.join(rhs.loc), arg_loc=lhs.loc, equals_loc=equals_loc) return thunk @action(Opt(Rule("comp_for"))) def argument_2(self, compose_opt): def thunk(lhs): if compose_opt: generators = compose_opt([]) return ast.GeneratorExp(elt=lhs, generators=generators, begin_loc=None, end_loc=None, loc=lhs.loc.join(generators[-1].loc)) return lhs return thunk @action(Seq(Rule("test"), Alt(argument_1, argument_2))) def argument(self, lhs, thunk): # This rule is reformulated to avoid exponential backtracking. """ (2.6) argument: test [gen_for] | test '=' test # Really [keyword '='] test (2.7-) argument: test [comp_for] | test '=' test """ return thunk(lhs) list_iter = Alt(Rule("list_for"), Rule("list_if")) """(2.6, 2.7) list_iter: list_for | list_if""" def list_comp_for_action(self, for_loc, target, in_loc, iter, next_opt): def compose(comprehensions): comp = ast.comprehension( target=target, iter=iter, ifs=[], loc=for_loc.join(iter.loc), for_loc=for_loc, in_loc=in_loc, if_locs=[]) comprehensions += [comp] if next_opt: return next_opt(comprehensions) else: return comprehensions return compose def list_comp_if_action(self, if_loc, cond, next_opt): def compose(comprehensions): comprehensions[-1].ifs.append(cond) comprehensions[-1].if_locs.append(if_loc) comprehensions[-1].loc = comprehensions[-1].loc.join(cond.loc) if next_opt: return next_opt(comprehensions) else: return comprehensions return compose list_for = action( Seq(Loc("for"), Rule("exprlist"), Loc("in"), Rule("testlist_safe"), Opt(Rule("list_iter")))) \ (list_comp_for_action) """(2.6, 2.7) list_for: 'for' exprlist 'in' testlist_safe [list_iter]""" list_if = action( Seq(Loc("if"), Rule("old_test"), Opt(Rule("list_iter")))) \ (list_comp_if_action) """(2.6, 2.7) list_if: 'if' old_test [list_iter]""" comp_iter = Alt(Rule("comp_for"), Rule("comp_if")) """ (2.6) gen_iter: gen_for | gen_if (2.7-) comp_iter: comp_for | comp_if """ comp_for = action( Seq(Loc("for"), Rule("exprlist"), Loc("in"), Rule("or_test"), Opt(Rule("comp_iter")))) \ (list_comp_for_action) """ (2.6) gen_for: 'for' exprlist 'in' or_test [gen_iter] (2.7-) comp_for: 'for' exprlist 'in' or_test [comp_iter] """ comp_if__26 = action( Seq(Loc("if"), Rule("old_test"), Opt(Rule("comp_iter")))) \ (list_comp_if_action) """ (2.6) gen_if: 'if' old_test [gen_iter] (2.7) comp_if: 'if' old_test [comp_iter] """ comp_if__30 = action( Seq(Loc("if"), Rule("test_nocond"), Opt(Rule("comp_iter")))) \ (list_comp_if_action) """ (3.0-) comp_if: 'if' test_nocond [comp_iter] """ testlist1 = action(List(Rule("test"), ",", trailing=False))(_wrap_tuple) """testlist1: test (',' test)*""" @action(Seq(Loc("yield"), Opt(Rule("testlist")))) def yield_expr__26(self, yield_loc, exprs): """(2.6, 2.7, 3.0, 3.1, 3.2) yield_expr: 'yield' [testlist]""" if exprs is not None: return ast.Yield(value=exprs, yield_loc=yield_loc, loc=yield_loc.join(exprs.loc)) else: return ast.Yield(value=None, yield_loc=yield_loc, loc=yield_loc) @action(Seq(Loc("yield"), Opt(Rule("yield_arg")))) def yield_expr__33(self, yield_loc, arg): """(3.3-) yield_expr: 'yield' [yield_arg]""" if isinstance(arg, ast.YieldFrom): arg.yield_loc = yield_loc arg.loc = arg.loc.join(arg.yield_loc) return arg elif arg is not None: return ast.Yield(value=arg, yield_loc=yield_loc, loc=yield_loc.join(arg.loc)) else: return ast.Yield(value=None, yield_loc=yield_loc, loc=yield_loc) @action(Seq(Loc("from"), Rule("test"))) def yield_arg_1(self, from_loc, value): return ast.YieldFrom(value=value, from_loc=from_loc, loc=from_loc.join(value.loc)) yield_arg = Alt(yield_arg_1, Rule("testlist")) """(3.3-) yield_arg: 'from' test | testlist""" ================================================ FILE: third_party/pythonparser/source.py ================================================ """ The :mod:`source` module concerns itself with manipulating buffers of source code: creating ranges of characters corresponding to a token, combining these ranges, extracting human-readable location information and original source from a range. """ from __future__ import absolute_import, division, print_function, unicode_literals import bisect import re class Buffer: """ A buffer containing source code and location information. :ivar source: (string) source code :ivar name: (string) input filename or another description of the input (e.g. ````). :ivar line: (integer) first line of the input """ def __init__(self, source, name="", first_line=1): self.encoding = self._extract_encoding(source) if isinstance(source, bytes): self.source = source.decode(self.encoding) else: self.source = source self.name = name self.first_line = first_line self._line_begins = None def __repr__(self): return "Buffer(\"%s\")" % self.name def source_line(self, lineno): """ Returns line ``lineno`` from source, taking ``first_line`` into account, or raises :exc:`IndexError` if ``lineno`` is out of range. """ line_begins = self._extract_line_begins() lineno = lineno - self.first_line if lineno >= 0 and lineno + 1 < len(line_begins): first, last = line_begins[lineno:lineno + 2] return self.source[first:last] elif lineno >= 0 and lineno < len(line_begins): return self.source[line_begins[-1]:] else: raise IndexError def decompose_position(self, offset): """ Returns a ``line, column`` tuple for a character offset into the source, orraises :exc:`IndexError` if ``lineno`` is out of range. """ line_begins = self._extract_line_begins() lineno = bisect.bisect_right(line_begins, offset) - 1 if offset >= 0 and offset <= len(self.source): return lineno + self.first_line, offset - line_begins[lineno] else: raise IndexError def _extract_line_begins(self): if self._line_begins: return self._line_begins self._line_begins = [0] index = None while True: index = self.source.find("\n", index) + 1 if index == 0: return self._line_begins self._line_begins.append(index) _encoding_re = re.compile("^[ \t\v]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)") _encoding_bytes_re = re.compile(_encoding_re.pattern.encode()) def _extract_encoding(self, source): if isinstance(source, bytes): re = self._encoding_bytes_re nl = b"\n" else: re = self._encoding_re nl = "\n" match = re.match(source) if not match: index = source.find(nl) if index != -1: match = re.match(source[index + 1:]) if match: encoding = match.group(1) if isinstance(encoding, bytes): return encoding.decode("ascii") return encoding return "ascii" class Range: """ Location of an exclusive range of characters [*begin_pos*, *end_pos*) in a :class:`Buffer`. :ivar begin_pos: (integer) offset of the first character :ivar end_pos: (integer) offset of the character before the last :ivar expanded_from: (Range or None) the range from which this range was expanded """ def __init__(self, source_buffer, begin_pos, end_pos, expanded_from=None): self.source_buffer = source_buffer self.begin_pos = begin_pos self.end_pos = end_pos self.expanded_from = expanded_from def __repr__(self): """ Returns a human-readable representation of this range. """ return "Range(\"%s\", %d, %d, %s)" % \ (self.source_buffer.name, self.begin_pos, self.end_pos, repr(self.expanded_from)) def chain(self, expanded_from): """ Returns a range identical to this one, but indicating that it was expanded from the range `expanded_from`. """ return Range(self.source_buffer, self.begin_pos, self.begin_pos, expanded_from=expanded_from) def begin(self): """ Returns a zero-length range located just before the beginning of this range. """ return Range(self.source_buffer, self.begin_pos, self.begin_pos, expanded_from=self.expanded_from) def end(self): """ Returns a zero-length range located just after the end of this range. """ return Range(self.source_buffer, self.end_pos, self.end_pos, expanded_from=self.expanded_from) def size(self): """ Returns the amount of characters spanned by the range. """ return self.end_pos - self.begin_pos def column(self): """ Returns a zero-based column number of the beginning of this range. """ line, column = self.source_buffer.decompose_position(self.begin_pos) return column def column_range(self): """ Returns a [*begin*, *end*) tuple describing the range of columns spanned by this range. If range spans more than one line, returned *end* is the last column of the line. """ if self.begin().line() == self.end().line(): return self.begin().column(), self.end().column() else: return self.begin().column(), len(self.begin().source_line()) - 1 def line(self): """ Returns the line number of the beginning of this range. """ line, column = self.source_buffer.decompose_position(self.begin_pos) return line def join(self, other): """ Returns the smallest possible range spanning both this range and other. Raises :exc:`ValueError` if the ranges do not belong to the same :class:`Buffer`. """ if self.source_buffer != other.source_buffer: raise ValueError if self.expanded_from == other.expanded_from: expanded_from = self.expanded_from else: expanded_from = None return Range(self.source_buffer, min(self.begin_pos, other.begin_pos), max(self.end_pos, other.end_pos), expanded_from=expanded_from) def source(self): """ Returns the source code covered by this range. """ return self.source_buffer.source[self.begin_pos:self.end_pos] def source_line(self): """ Returns the line of source code containing the beginning of this range. """ return self.source_buffer.source_line(self.line()) def source_lines(self): """ Returns the lines of source code containing the entirety of this range. """ return [self.source_buffer.source_line(line) for line in range(self.line(), self.end().line() + 1)] def __str__(self): """ Returns a Clang-style string representation of the beginning of this range. """ if self.begin_pos != self.end_pos: return "%s:%d:%d-%d:%d" % (self.source_buffer.name, self.line(), self.column() + 1, self.end().line(), self.end().column() + 1) else: return "%s:%d:%d" % (self.source_buffer.name, self.line(), self.column() + 1) def __eq__(self, other): """ Returns true if the ranges have the same source buffer, start and end position. """ return (type(self) == type(other) and self.source_buffer == other.source_buffer and self.begin_pos == other.begin_pos and self.end_pos == other.end_pos and self.expanded_from == other.expanded_from) def __ne__(self, other): """ Inverse of :meth:`__eq__`. """ return not (self == other) def __hash__(self): return hash((self.source_buffer, self.begin_pos, self.end_pos, self.expanded_from)) class Comment: """ A comment in the source code. :ivar loc: (:class:`Range`) source location :ivar text: (string) comment text """ def __init__(self, loc, text): self.loc, self.text = loc, text class RewriterConflict(Exception): """ An exception that is raised when two ranges supplied to a rewriter overlap. :ivar first: (:class:`Range`) first overlapping range :ivar second: (:class:`Range`) second overlapping range """ def __init__(self, first, second): self.first, self.second = first, second exception.__init__(self, "Ranges %s and %s overlap" % (repr(first), repr(second))) class Rewriter: """ The :class:`Rewriter` class rewrites source code: performs bulk modification guided by a list of ranges and code fragments replacing their original content. :ivar buffer: (:class:`Buffer`) buffer """ def __init__(self, buffer): self.buffer = buffer self.ranges = [] def replace(self, range, replacement): """Remove `range` and replace it with string `replacement`.""" self.ranges.append((range, replacement)) def remove(self, range): """Remove `range`.""" self.replace(range, "") def insert_before(self, range, text): """Insert `text` before `range`.""" self.replace(range.begin(), text) def insert_after(self, range, text): """Insert `text` after `range`.""" self.replace(range.end(), text) def rewrite(self): """Return the rewritten source. May raise :class:`RewriterConflict`.""" self._sort() self._check() rewritten, pos = [], 0 for range, replacement in self.ranges: rewritten.append(self.buffer.source[pos:range.begin_pos]) rewritten.append(replacement) pos = range.end_pos rewritten.append(self.buffer.source[pos:]) return Buffer("".join(rewritten), self.buffer.name, self.buffer.first_line) def _sort(self): self.ranges.sort(key=lambda x: x[0].begin_pos) def _check(self): for (fst, _), (snd, _) in zip(self.ranges, self.ranges[1:]): if snd.begin_pos < fst.end_pos: raise RewriterConflict(fst, snd) ================================================ FILE: third_party/stdlib/LICENSE ================================================ A. HISTORY OF THE SOFTWARE ========================== Python was created in the early 1990s by Guido van Rossum at Stichting Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands as a successor of a language called ABC. Guido remains Python's principal author, although it includes many contributions from others. In 1995, Guido continued his work on Python at the Corporation for National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) in Reston, Virginia where he released several versions of the software. In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same year, the PythonLabs team moved to Digital Creations (now Zope Corporation, see http://www.zope.com). In 2001, the Python Software Foundation (PSF, see http://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation is a sponsoring member of the PSF. All Python releases are Open Source (see http://www.opensource.org for the Open Source Definition). Historically, most, but not all, Python releases have also been GPL-compatible; the table below summarizes the various releases. Release Derived Year Owner GPL- from compatible? (1) 0.9.0 thru 1.2 1991-1995 CWI yes 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes 1.6 1.5.2 2000 CNRI no 2.0 1.6 2000 BeOpen.com no 1.6.1 1.6 2001 CNRI yes (2) 2.1 2.0+1.6.1 2001 PSF no 2.0.1 2.0+1.6.1 2001 PSF yes 2.1.1 2.1+2.0.1 2001 PSF yes 2.1.2 2.1.1 2002 PSF yes 2.1.3 2.1.2 2002 PSF yes 2.2 and above 2.1.1 2001-now PSF yes Footnotes: (1) GPL-compatible doesn't mean that we're distributing Python under the GPL. All Python licenses, unlike the GPL, let you distribute a modified version without making your changes open source. The GPL-compatible licenses make it possible to combine Python with other software that is released under the GPL; the others don't. (2) According to Richard Stallman, 1.6.1 is not GPL-compatible, because its license has a choice of law clause. According to CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 is "not incompatible" with the GPL. Thanks to the many outside volunteers who have worked under Guido's direction to make these releases possible. B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON =============================================================== PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 -------------------------------------------- 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python. 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 ------------------------------------------- BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the Individual or Organization ("Licensee") accessing and otherwise using this software in source or binary form and its associated documentation ("the Software"). 2. Subject to the terms and conditions of this BeOpen Python License Agreement, BeOpen hereby grants Licensee a non-exclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use the Software alone or in any derivative version, provided, however, that the BeOpen Python License is retained in the Software, alone or in any derivative version prepared by Licensee. 3. BeOpen is making the Software available to Licensee on an "AS IS" basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 5. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 6. This License Agreement shall be governed by and interpreted in all respects by the law of the State of California, excluding conflict of law provisions. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between BeOpen and Licensee. This License Agreement does not grant permission to use BeOpen trademarks or trade names in a trademark sense to endorse or promote products or services of Licensee, or any third party. As an exception, the "BeOpen Python" logos available at http://www.pythonlabs.com/logos.html may be used according to the permissions granted on that web page. 7. By copying, installing or otherwise using the software, Licensee agrees to be bound by the terms and conditions of this License Agreement. CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 --------------------------------------- 1. This LICENSE AGREEMENT is between the Corporation for National Research Initiatives, having an office at 1895 Preston White Drive, Reston, VA 20191 ("CNRI"), and the Individual or Organization ("Licensee") accessing and otherwise using Python 1.6.1 software in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, CNRI hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python 1.6.1 alone or in any derivative version, provided, however, that CNRI's License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) 1995-2001 Corporation for National Research Initiatives; All Rights Reserved" are retained in Python 1.6.1 alone or in any derivative version prepared by Licensee. Alternately, in lieu of CNRI's License Agreement, Licensee may substitute the following text (omitting the quotes): "Python 1.6.1 is made available subject to the terms and conditions in CNRI's License Agreement. This Agreement together with Python 1.6.1 may be located on the Internet using the following unique, persistent identifier (known as a handle): 1895.22/1013. This Agreement may also be obtained from a proxy server on the Internet using the following URL: http://hdl.handle.net/1895.22/1013". 3. In the event Licensee prepares a derivative work that is based on or incorporates Python 1.6.1 or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python 1.6.1. 4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. This License Agreement shall be governed by the federal intellectual property law of the United States, including without limitation the federal copyright law, and, to the extent such U.S. federal law does not apply, by the law of the Commonwealth of Virginia, excluding Virginia's conflict of law provisions. Notwithstanding the foregoing, with regard to derivative works based on Python 1.6.1 that incorporate non-separable material that was previously distributed under the GNU General Public License (GPL), the law of the Commonwealth of Virginia shall govern this License Agreement only as to issues arising under or with respect to Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between CNRI and Licensee. This License Agreement does not grant permission to use CNRI trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By clicking on the "ACCEPT" button where indicated, or by copying, installing or otherwise using Python 1.6.1, Licensee agrees to be bound by the terms and conditions of this License Agreement. ACCEPT CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 -------------------------------------------------- Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, The Netherlands. All rights reserved. Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of Stichting Mathematisch Centrum or CWI not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ================================================ FILE: third_party/stdlib/Queue.py ================================================ """A multi-producer, multi-consumer queue.""" from time import time as _time #try: import threading as _threading #except ImportError: # import dummy_threading as _threading from collections import deque import heapq __all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue'] class Empty(Exception): "Exception raised by Queue.get(block=0)/get_nowait()." pass class Full(Exception): "Exception raised by Queue.put(block=0)/put_nowait()." pass class Queue(object): """Create a queue object with a given maximum size. If maxsize is <= 0, the queue size is infinite. """ def __init__(self, maxsize=0): self.maxsize = maxsize self._init(maxsize) # mutex must be held whenever the queue is mutating. All methods # that acquire mutex must release it before returning. mutex # is shared between the three conditions, so acquiring and # releasing the conditions also acquires and releases mutex. self.mutex = _threading.Lock() # Notify not_empty whenever an item is added to the queue; a # thread waiting to get is notified then. self.not_empty = _threading.Condition(self.mutex) # Notify not_full whenever an item is removed from the queue; # a thread waiting to put is notified then. self.not_full = _threading.Condition(self.mutex) # Notify all_tasks_done whenever the number of unfinished tasks # drops to zero; thread waiting to join() is notified to resume self.all_tasks_done = _threading.Condition(self.mutex) self.unfinished_tasks = 0 def task_done(self): """Indicate that a formerly enqueued task is complete. Used by Queue consumer threads. For each get() used to fetch a task, a subsequent call to task_done() tells the queue that the processing on the task is complete. If a join() is currently blocking, it will resume when all items have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). Raises a ValueError if called more times than there were items placed in the queue. """ self.all_tasks_done.acquire() try: unfinished = self.unfinished_tasks - 1 if unfinished <= 0: if unfinished < 0: raise ValueError('task_done() called too many times') self.all_tasks_done.notify_all() self.unfinished_tasks = unfinished finally: self.all_tasks_done.release() def join(self): """Blocks until all items in the Queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer thread calls task_done() to indicate the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, join() unblocks. """ self.all_tasks_done.acquire() try: while self.unfinished_tasks: self.all_tasks_done.wait() finally: self.all_tasks_done.release() def qsize(self): """Return the approximate size of the queue (not reliable!).""" self.mutex.acquire() n = self._qsize() self.mutex.release() return n def empty(self): """Return True if the queue is empty, False otherwise (not reliable!).""" self.mutex.acquire() n = not self._qsize() self.mutex.release() return n def full(self): """Return True if the queue is full, False otherwise (not reliable!).""" self.mutex.acquire() n = 0 < self.maxsize == self._qsize() self.mutex.release() return n def put(self, item, block=True, timeout=None): """Put an item into the queue. If optional args 'block' is true and 'timeout' is None (the default), block if necessary until a free slot is available. If 'timeout' is a non-negative number, it blocks at most 'timeout' seconds and raises the Full exception if no free slot was available within that time. Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). """ self.not_full.acquire() try: if self.maxsize > 0: if not block: if self._qsize() == self.maxsize: raise Full elif timeout is None: while self._qsize() == self.maxsize: self.not_full.wait() elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = _time() + timeout while self._qsize() == self.maxsize: remaining = endtime - _time() if remaining <= 0.0: raise Full self.not_full.wait(remaining) self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() finally: self.not_full.release() def put_nowait(self, item): """Put an item into the queue without blocking. Only enqueue the item if a free slot is immediately available. Otherwise raise the Full exception. """ return self.put(item, False) def get(self, block=True, timeout=None): """Remove and return an item from the queue. If optional args 'block' is true and 'timeout' is None (the default), block if necessary until an item is available. If 'timeout' is a non-negative number, it blocks at most 'timeout' seconds and raises the Empty exception if no item was available within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). """ self.not_empty.acquire() try: if not block: if not self._qsize(): raise Empty elif timeout is None: while not self._qsize(): self.not_empty.wait() elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: endtime = _time() + timeout while not self._qsize(): remaining = endtime - _time() if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) item = self._get() self.not_full.notify() return item finally: self.not_empty.release() def get_nowait(self): """Remove and return an item from the queue without blocking. Only get an item if one is immediately available. Otherwise raise the Empty exception. """ return self.get(False) # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held # Initialize the queue representation def _init(self, maxsize): self.queue = deque() def _qsize(self, len=len): return len(self.queue) # Put a new item in the queue def _put(self, item): self.queue.append(item) # Get an item from the queue def _get(self): return self.queue.popleft() class PriorityQueue(Queue): '''Variant of Queue that retrieves open entries in priority order (lowest first). Entries are typically tuples of the form: (priority number, data). ''' def _init(self, maxsize): self.queue = [] def _qsize(self, len=len): return len(self.queue) def _put(self, item, heappush=heapq.heappush): heappush(self.queue, item) def _get(self, heappop=heapq.heappop): return heappop(self.queue) class LifoQueue(Queue): '''Variant of Queue that retrieves most recently added entries first.''' def _init(self, maxsize): self.queue = [] def _qsize(self, len=len): return len(self.queue) def _put(self, item): self.queue.append(item) def _get(self): return self.queue.pop() ================================================ FILE: third_party/stdlib/README.md ================================================ Canonical versions of the files in this folder come from the [Lib](https://github.com/python/cpython/tree/2.7/Lib) directory of the [2.7 branch of the CPython repo](https://github.com/python/cpython/tree/2.7). ================================================ FILE: third_party/stdlib/StringIO.py ================================================ r"""File-like objects that read from or write to a string buffer. This implements (nearly) all stdio methods. f = StringIO() # ready for writing f = StringIO(buf) # ready for reading f.close() # explicitly release resources held flag = f.isatty() # always false pos = f.tell() # get current position f.seek(pos) # set current position f.seek(pos, mode) # mode 0: absolute; 1: relative; 2: relative to EOF buf = f.read() # read until EOF buf = f.read(n) # read up to n bytes buf = f.readline() # read until end of line ('\n') or EOF list = f.readlines()# list of f.readline() results until EOF f.truncate([size]) # truncate file at to at most size (default: current pos) f.write(buf) # write at current position f.writelines(list) # for line in list: f.write(line) f.getvalue() # return whole file's contents as a string Notes: - Using a real file is often faster (but less convenient). - There's also a much faster implementation in C, called cStringIO, but it's not subclassable. - fileno() is left unimplemented so that code which uses it triggers an exception early. - Seeking far beyond EOF and then writing will insert real null bytes that occupy space in the buffer. - There's a simple test set (see end of this file). """ try: import errno EINVAL = errno.EINVAL except ImportError: EINVAL = 22 __all__ = ["StringIO"] def _complain_ifclosed(closed): if closed: raise ValueError, "I/O operation on closed file" class StringIO(object): """class StringIO([buffer]) When a StringIO object is created, it can be initialized to an existing string by passing the string to the constructor. If no string is given, the StringIO will start empty. The StringIO object can accept either Unicode or 8-bit strings, but mixing the two may take some care. If both are used, 8-bit strings that cannot be interpreted as 7-bit ASCII (that use the 8th bit) will cause a UnicodeError to be raised when getvalue() is called. """ def __init__(self, buf = ''): # Force self.buf to be a string or unicode if not isinstance(buf, basestring): buf = str(buf) self.buf = buf self.len = len(buf) self.buflist = [] self.pos = 0 self.closed = False self.softspace = 0 def __iter__(self): return self def next(self): """A file object is its own iterator, for example iter(f) returns f (unless f is closed). When a file is used as an iterator, typically in a for loop (for example, for line in f: print line), the next() method is called repeatedly. This method returns the next input line, or raises StopIteration when EOF is hit. """ _complain_ifclosed(self.closed) r = self.readline() if not r: raise StopIteration return r def close(self): """Free the memory buffer. """ if not self.closed: self.closed = True del self.buf, self.pos def isatty(self): """Returns False because StringIO objects are not connected to a tty-like device. """ _complain_ifclosed(self.closed) return False def seek(self, pos, mode = 0): """Set the file's current position. The mode argument is optional and defaults to 0 (absolute file positioning); other values are 1 (seek relative to the current position) and 2 (seek relative to the file's end). There is no return value. """ _complain_ifclosed(self.closed) if self.buflist: self.buf += ''.join(self.buflist) self.buflist = [] if mode == 1: pos += self.pos elif mode == 2: pos += self.len self.pos = max(0, pos) def tell(self): """Return the file's current position.""" _complain_ifclosed(self.closed) return self.pos def read(self, n = -1): """Read at most size bytes from the file (less if the read hits EOF before obtaining size bytes). If the size argument is negative or omitted, read all data until EOF is reached. The bytes are returned as a string object. An empty string is returned when EOF is encountered immediately. """ _complain_ifclosed(self.closed) if self.buflist: self.buf += ''.join(self.buflist) self.buflist = [] if n is None or n < 0: newpos = self.len else: newpos = min(self.pos+n, self.len) r = self.buf[self.pos:newpos] self.pos = newpos return r def readline(self, length=None): r"""Read one entire line from the file. A trailing newline character is kept in the string (but may be absent when a file ends with an incomplete line). If the size argument is present and non-negative, it is a maximum byte count (including the trailing newline) and an incomplete line may be returned. An empty string is returned only when EOF is encountered immediately. Note: Unlike stdio's fgets(), the returned string contains null characters ('\0') if they occurred in the input. """ _complain_ifclosed(self.closed) if self.buflist: self.buf += ''.join(self.buflist) self.buflist = [] i = self.buf.find('\n', self.pos) if i < 0: newpos = self.len else: newpos = i+1 if length is not None and length >= 0: if self.pos + length < newpos: newpos = self.pos + length r = self.buf[self.pos:newpos] self.pos = newpos return r def readlines(self, sizehint = 0): """Read until EOF using readline() and return a list containing the lines thus read. If the optional sizehint argument is present, instead of reading up to EOF, whole lines totalling approximately sizehint bytes (or more to accommodate a final whole line). """ total = 0 lines = [] line = self.readline() while line: lines.append(line) total += len(line) if 0 < sizehint <= total: break line = self.readline() return lines def truncate(self, size=None): """Truncate the file's size. If the optional size argument is present, the file is truncated to (at most) that size. The size defaults to the current position. The current file position is not changed unless the position is beyond the new file size. If the specified size exceeds the file's current size, the file remains unchanged. """ _complain_ifclosed(self.closed) if size is None: size = self.pos elif size < 0: raise IOError(EINVAL, "Negative size not allowed") elif size < self.pos: self.pos = size self.buf = self.getvalue()[:size] self.len = size def write(self, s): """Write a string to the file. There is no return value. """ _complain_ifclosed(self.closed) if not s: return # Force s to be a string or unicode if not isinstance(s, basestring): s = str(s) spos = self.pos slen = self.len if spos == slen: self.buflist.append(s) self.len = self.pos = spos + len(s) return if spos > slen: self.buflist.append('\0'*(spos - slen)) slen = spos newpos = spos + len(s) if spos < slen: if self.buflist: self.buf += ''.join(self.buflist) self.buflist = [self.buf[:spos], s, self.buf[newpos:]] self.buf = '' if newpos > slen: slen = newpos else: self.buflist.append(s) slen = newpos self.len = slen self.pos = newpos def writelines(self, iterable): """Write a sequence of strings to the file. The sequence can be any iterable object producing strings, typically a list of strings. There is no return value. (The name is intended to match readlines(); writelines() does not add line separators.) """ write = self.write for line in iterable: write(line) def flush(self): """Flush the internal buffer """ _complain_ifclosed(self.closed) def getvalue(self): """ Retrieve the entire contents of the "file" at any time before the StringIO object's close() method is called. The StringIO object can accept either Unicode or 8-bit strings, but mixing the two may take some care. If both are used, 8-bit strings that cannot be interpreted as 7-bit ASCII (that use the 8th bit) will cause a UnicodeError to be raised when getvalue() is called. """ _complain_ifclosed(self.closed) if self.buflist: self.buf += ''.join(self.buflist) self.buflist = [] return self.buf # A little test suite def test(): import sys if sys.argv[1:]: file = sys.argv[1] else: file = '/etc/passwd' lines = open(file, 'r').readlines() text = open(file, 'r').read() f = StringIO() for line in lines[:-2]: f.write(line) f.writelines(lines[-2:]) if f.getvalue() != text: raise RuntimeError, 'write failed' length = f.tell() print 'File length =', length f.seek(len(lines[0])) f.write(lines[1]) f.seek(0) print 'First line =', repr(f.readline()) print 'Position =', f.tell() line = f.readline() print 'Second line =', repr(line) f.seek(-len(line), 1) line2 = f.read(len(line)) if line != line2: raise RuntimeError, 'bad result after seek back' f.seek(len(line2), 1) list = f.readlines() line = list[-1] f.seek(f.tell() - len(line)) line2 = f.read() if line != line2: raise RuntimeError, 'bad result after seek back from EOF' print 'Read', len(list), 'more lines' print 'File length =', f.tell() if f.tell() != length: raise RuntimeError, 'bad length' f.truncate(length/2) f.seek(0, 2) print 'Truncated length =', f.tell() if f.tell() != length/2: raise RuntimeError, 'truncate did not adjust length' f.close() if __name__ == '__main__': test() ================================================ FILE: third_party/stdlib/UserDict.py ================================================ """A more or less complete user-defined wrapper around dictionary objects.""" class UserDict(object): def __init__(*args, **kwargs): if not args: raise TypeError("descriptor '__init__' of 'UserDict' object " "needs an argument") self = args[0] args = args[1:] if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) if args: dict = args[0] elif 'dict' in kwargs: dict = kwargs.pop('dict') import warnings warnings.warn("Passing 'dict' as keyword argument is " "deprecated", PendingDeprecationWarning, stacklevel=2) else: dict = None self.data = {} if dict is not None: self.update(dict) if len(kwargs): self.update(kwargs) def __repr__(self): return repr(self.data) def __cmp__(self, dict): if isinstance(dict, UserDict): return cmp(self.data, dict.data) else: return cmp(self.data, dict) __hash__ = None # Avoid Py3k warning def __len__(self): return len(self.data) def __getitem__(self, key): if key in self.data: return self.data[key] if hasattr(self.__class__, "__missing__"): return self.__class__.__missing__(self, key) raise KeyError(key) def __setitem__(self, key, item): self.data[key] = item def __delitem__(self, key): del self.data[key] def clear(self): self.data.clear() def copy(self): if self.__class__ is UserDict: return UserDict(self.data.copy()) import copy data = self.data try: self.data = {} c = copy.copy(self) finally: self.data = data c.update(self) return c def keys(self): return self.data.keys() def items(self): return self.data.items() def iteritems(self): return self.data.iteritems() def iterkeys(self): return self.data.iterkeys() def itervalues(self): return self.data.itervalues() def values(self): return self.data.values() def has_key(self, key): return key in self.data def update(*args, **kwargs): if not args: raise TypeError("descriptor 'update' of 'UserDict' object " "needs an argument") self = args[0] args = args[1:] if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) if args: dict = args[0] elif 'dict' in kwargs: dict = kwargs.pop('dict') import warnings warnings.warn("Passing 'dict' as keyword argument is deprecated", PendingDeprecationWarning, stacklevel=2) else: dict = None if dict is None: pass elif isinstance(dict, UserDict): self.data.update(dict.data) elif isinstance(dict, type({})) or not hasattr(dict, 'items'): self.data.update(dict) else: for k, v in dict.items(): self[k] = v if len(kwargs): self.data.update(kwargs) def get(self, key, failobj=None): if key not in self: return failobj return self[key] def setdefault(self, key, failobj=None): if key not in self: self[key] = failobj return self[key] def pop(self, key, *args): return self.data.pop(key, *args) def popitem(self): return self.data.popitem() def __contains__(self, key): return key in self.data def fromkeys(cls, iterable, value=None): d = cls() for key in iterable: d[key] = value return d # TODO: Make this a decorator once they're implemented. fromkeys = classmethod(fromkeys) class IterableUserDict(UserDict): def __iter__(self): return iter(self.data) import _abcoll _abcoll.MutableMapping.register(IterableUserDict) class DictMixin(object): # Mixin defining all dictionary methods for classes that already have # a minimum dictionary interface including getitem, setitem, delitem, # and keys. Without knowledge of the subclass constructor, the mixin # does not define __init__() or copy(). In addition to the four base # methods, progressively more efficiency comes with defining # __contains__(), __iter__(), and iteritems(). # second level definitions support higher levels def __iter__(self): for k in self.keys(): yield k def has_key(self, key): try: self[key] except KeyError: return False return True def __contains__(self, key): return self.has_key(key) # third level takes advantage of second level definitions def iteritems(self): for k in self: yield (k, self[k]) def iterkeys(self): return self.__iter__() # fourth level uses definitions from lower levels def itervalues(self): for _, v in self.iteritems(): yield v def values(self): return [v for _, v in self.iteritems()] def items(self): return list(self.iteritems()) def clear(self): for key in self.keys(): del self[key] def setdefault(self, key, default=None): try: return self[key] except KeyError: self[key] = default return default def pop(self, key, *args): if len(args) > 1: raise TypeError, "pop expected at most 2 arguments, got "\ + repr(1 + len(args)) try: value = self[key] except KeyError: if args: return args[0] raise del self[key] return value def popitem(self): try: k, v = self.iteritems().next() except StopIteration: raise KeyError, 'container is empty' del self[k] return (k, v) def update(self, other=None, **kwargs): # Make progressively weaker assumptions about "other" if other is None: pass elif hasattr(other, 'iteritems'): # iteritems saves memory and lookups for k, v in other.iteritems(): self[k] = v elif hasattr(other, 'keys'): for k in other.keys(): self[k] = other[k] else: for k, v in other: self[k] = v if kwargs: self.update(kwargs) def get(self, key, default=None): try: return self[key] except KeyError: return default def __repr__(self): return repr(dict(self.iteritems())) def __cmp__(self, other): if other is None: return 1 if isinstance(other, DictMixin): other = dict(other.iteritems()) return cmp(dict(self.iteritems()), other) def __len__(self): return len(self.keys()) ================================================ FILE: third_party/stdlib/UserList.py ================================================ """A more or less complete user-defined wrapper around list objects.""" import collections class UserList(collections.MutableSequence): def __init__(self, initlist=None): self.data = [] if initlist is not None: # XXX should this accept an arbitrary sequence? if type(initlist) == type(self.data): self.data[:] = initlist elif isinstance(initlist, UserList): self.data[:] = initlist.data[:] else: self.data = list(initlist) def __repr__(self): return repr(self.data) def __lt__(self, other): return self.data < self.__cast(other) def __le__(self, other): return self.data <= self.__cast(other) def __eq__(self, other): return self.data == self.__cast(other) def __ne__(self, other): return self.data != self.__cast(other) def __gt__(self, other): return self.data > self.__cast(other) def __ge__(self, other): return self.data >= self.__cast(other) def __cast(self, other): if isinstance(other, UserList): return other.data else: return other def __cmp__(self, other): return cmp(self.data, self.__cast(other)) __hash__ = None # Mutable sequence, so not hashable def __contains__(self, item): return item in self.data def __len__(self): return len(self.data) def __getitem__(self, i): return self.data[i] def __setitem__(self, i, item): self.data[i] = item def __delitem__(self, i): del self.data[i] def __getslice__(self, i, j): i = max(i, 0); j = max(j, 0) return self.__class__(self.data[i:j]) def __setslice__(self, i, j, other): i = max(i, 0); j = max(j, 0) if isinstance(other, UserList): self.data[i:j] = other.data elif isinstance(other, type(self.data)): self.data[i:j] = other else: self.data[i:j] = list(other) def __delslice__(self, i, j): i = max(i, 0); j = max(j, 0) del self.data[i:j] def __add__(self, other): if isinstance(other, UserList): return self.__class__(self.data + other.data) elif isinstance(other, type(self.data)): return self.__class__(self.data + other) else: return self.__class__(self.data + list(other)) def __radd__(self, other): if isinstance(other, UserList): return self.__class__(other.data + self.data) elif isinstance(other, type(self.data)): return self.__class__(other + self.data) else: return self.__class__(list(other) + self.data) def __iadd__(self, other): if isinstance(other, UserList): self.data += other.data elif isinstance(other, type(self.data)): self.data += other else: self.data += list(other) return self def __mul__(self, n): return self.__class__(self.data*n) __rmul__ = __mul__ def __imul__(self, n): self.data *= n return self def append(self, item): self.data.append(item) def insert(self, i, item): self.data.insert(i, item) def pop(self, i=-1): return self.data.pop(i) def remove(self, item): self.data.remove(item) def count(self, item): return self.data.count(item) def index(self, item, *args): return self.data.index(item, *args) def reverse(self): self.data.reverse() def sort(self, *args, **kwds): self.data.sort(*args, **kwds) def extend(self, other): if isinstance(other, UserList): self.data.extend(other.data) else: self.data.extend(other) ================================================ FILE: third_party/stdlib/UserString.py ================================================ #!/usr/bin/env python ## vim:ts=4:et:nowrap """A user-defined wrapper around string objects Note: string objects have grown methods in Python 1.6 This module requires Python 1.6 or later. """ import sys import collections __all__ = ["UserString","MutableString"] class UserString(collections.Sequence): def __init__(self, seq): if isinstance(seq, basestring): self.data = seq elif isinstance(seq, UserString): self.data = seq.data[:] else: self.data = str(seq) def __str__(self): return str(self.data) def __repr__(self): return repr(self.data) def __int__(self): return int(self.data) def __long__(self): return long(self.data) def __float__(self): return float(self.data) def __complex__(self): return complex(self.data) def __hash__(self): return hash(self.data) def __cmp__(self, string): if isinstance(string, UserString): return cmp(self.data, string.data) else: return cmp(self.data, string) def __contains__(self, char): return char in self.data def __len__(self): return len(self.data) def __getitem__(self, index): return self.__class__(self.data[index]) def __getslice__(self, start, end): start = max(start, 0); end = max(end, 0) return self.__class__(self.data[start:end]) def __add__(self, other): if isinstance(other, UserString): return self.__class__(self.data + other.data) elif isinstance(other, basestring): return self.__class__(self.data + other) else: return self.__class__(self.data + str(other)) def __radd__(self, other): if isinstance(other, basestring): return self.__class__(other + self.data) else: return self.__class__(str(other) + self.data) def __mul__(self, n): return self.__class__(self.data*n) __rmul__ = __mul__ def __mod__(self, args): return self.__class__(self.data % args) # the following methods are defined in alphabetical order: def capitalize(self): return self.__class__(self.data.capitalize()) def center(self, width, *args): return self.__class__(self.data.center(width, *args)) def count(self, sub, start=0, end=sys.maxint): return self.data.count(sub, start, end) def decode(self, encoding=None, errors=None): # XXX improve this? if encoding: if errors: return self.__class__(self.data.decode(encoding, errors)) else: return self.__class__(self.data.decode(encoding)) else: return self.__class__(self.data.decode()) def encode(self, encoding=None, errors=None): # XXX improve this? if encoding: if errors: return self.__class__(self.data.encode(encoding, errors)) else: return self.__class__(self.data.encode(encoding)) else: return self.__class__(self.data.encode()) def endswith(self, suffix, start=0, end=sys.maxint): return self.data.endswith(suffix, start, end) def expandtabs(self, tabsize=8): return self.__class__(self.data.expandtabs(tabsize)) def find(self, sub, start=0, end=sys.maxint): return self.data.find(sub, start, end) def index(self, sub, start=0, end=sys.maxint): return self.data.index(sub, start, end) def isalpha(self): return self.data.isalpha() def isalnum(self): return self.data.isalnum() def isdecimal(self): return self.data.isdecimal() def isdigit(self): return self.data.isdigit() def islower(self): return self.data.islower() def isnumeric(self): return self.data.isnumeric() def isspace(self): return self.data.isspace() def istitle(self): return self.data.istitle() def isupper(self): return self.data.isupper() def join(self, seq): return self.data.join(seq) def ljust(self, width, *args): return self.__class__(self.data.ljust(width, *args)) def lower(self): return self.__class__(self.data.lower()) def lstrip(self, chars=None): return self.__class__(self.data.lstrip(chars)) def partition(self, sep): return self.data.partition(sep) def replace(self, old, new, maxsplit=-1): return self.__class__(self.data.replace(old, new, maxsplit)) def rfind(self, sub, start=0, end=sys.maxint): return self.data.rfind(sub, start, end) def rindex(self, sub, start=0, end=sys.maxint): return self.data.rindex(sub, start, end) def rjust(self, width, *args): return self.__class__(self.data.rjust(width, *args)) def rpartition(self, sep): return self.data.rpartition(sep) def rstrip(self, chars=None): return self.__class__(self.data.rstrip(chars)) def split(self, sep=None, maxsplit=-1): return self.data.split(sep, maxsplit) def rsplit(self, sep=None, maxsplit=-1): return self.data.rsplit(sep, maxsplit) def splitlines(self, keepends=0): return self.data.splitlines(keepends) def startswith(self, prefix, start=0, end=sys.maxint): return self.data.startswith(prefix, start, end) def strip(self, chars=None): return self.__class__(self.data.strip(chars)) def swapcase(self): return self.__class__(self.data.swapcase()) def title(self): return self.__class__(self.data.title()) def translate(self, *args): return self.__class__(self.data.translate(*args)) def upper(self): return self.__class__(self.data.upper()) def zfill(self, width): return self.__class__(self.data.zfill(width)) class MutableString(UserString, collections.MutableSequence): """mutable string objects Python strings are immutable objects. This has the advantage, that strings may be used as dictionary keys. If this property isn't needed and you insist on changing string values in place instead, you may cheat and use MutableString. But the purpose of this class is an educational one: to prevent people from inventing their own mutable string class derived from UserString and than forget thereby to remove (override) the __hash__ method inherited from UserString. This would lead to errors that would be very hard to track down. A faster and better solution is to rewrite your program using lists.""" def __init__(self, string=""): # from warnings import warnpy3k import warnings warnpy3k = warnings.warnpy3k warnpy3k('the class UserString.MutableString has been removed in ' 'Python 3.0', stacklevel=2) self.data = string # We inherit object.__hash__, so we must deny this explicitly __hash__ = None def __setitem__(self, index, sub): if isinstance(index, slice): if isinstance(sub, UserString): sub = sub.data elif not isinstance(sub, basestring): sub = str(sub) start, stop, step = index.indices(len(self.data)) if step == -1: start, stop = stop+1, start+1 sub = sub[::-1] elif step != 1: # XXX(twouters): I guess we should be reimplementing # the extended slice assignment/deletion algorithm here... raise TypeError, "invalid step in slicing assignment" start = min(start, stop) self.data = self.data[:start] + sub + self.data[stop:] else: if index < 0: index += len(self.data) if index < 0 or index >= len(self.data): raise IndexError self.data = self.data[:index] + sub + self.data[index+1:] def __delitem__(self, index): if isinstance(index, slice): start, stop, step = index.indices(len(self.data)) if step == -1: start, stop = stop+1, start+1 elif step != 1: # XXX(twouters): see same block in __setitem__ raise TypeError, "invalid step in slicing deletion" start = min(start, stop) self.data = self.data[:start] + self.data[stop:] else: if index < 0: index += len(self.data) if index < 0 or index >= len(self.data): raise IndexError self.data = self.data[:index] + self.data[index+1:] def __setslice__(self, start, end, sub): start = max(start, 0); end = max(end, 0) if isinstance(sub, UserString): self.data = self.data[:start]+sub.data+self.data[end:] elif isinstance(sub, basestring): self.data = self.data[:start]+sub+self.data[end:] else: self.data = self.data[:start]+str(sub)+self.data[end:] def __delslice__(self, start, end): start = max(start, 0); end = max(end, 0) self.data = self.data[:start] + self.data[end:] def immutable(self): return UserString(self.data) def __iadd__(self, other): if isinstance(other, UserString): self.data += other.data elif isinstance(other, basestring): self.data += other else: self.data += str(other) return self def __imul__(self, n): self.data *= n return self def insert(self, index, value): self[index:index] = value # if __name__ == "__main__": # # execute the regression test to stdout, if called as a script: # import os # called_in_dir, called_as = os.path.split(sys.argv[0]) # called_as, py = os.path.splitext(called_as) # if '-q' in sys.argv: # from test import test_support # test_support.verbose = 0 # __import__('test.test_' + called_as.lower()) ================================================ FILE: third_party/stdlib/_abcoll.py ================================================ # Copyright 2007 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Abstract Base Classes (ABCs) for collections, according to PEP 3119. DON'T USE THIS MODULE DIRECTLY! The classes here should be imported via collections; they are defined here only to alleviate certain bootstrapping issues. Unit tests are in test_collections. """ import abc ABCMeta = abc.ABCMeta abstractmethod = abc.abstractmethod import sys __all__ = ["Hashable", "Iterable", "Iterator", "Sized", "Container", "Callable", "Set", "MutableSet", "Mapping", "MutableMapping", "MappingView", #"KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", ] ### ONE-TRICK PONIES ### def _hasattr(C, attr): try: return any(attr in B.__dict__ for B in C.__mro__) except AttributeError: # Old-style class return hasattr(C, attr) class Hashable(object): __metaclass__ = ABCMeta @abstractmethod def __hash__(self): return 0 @classmethod def __subclasshook__(cls, C): if cls is Hashable: try: for B in C.__mro__: if "__hash__" in B.__dict__: if B.__dict__["__hash__"]: return True break except AttributeError: # Old-style class if getattr(C, "__hash__", None): return True return NotImplemented class Iterable(object): __metaclass__ = ABCMeta @abstractmethod def __iter__(self): while False: yield None @classmethod def __subclasshook__(cls, C): if cls is Iterable: if _hasattr(C, "__iter__"): return True return NotImplemented Iterable.register(str) class Iterator(Iterable): @abstractmethod def next(self): 'Return the next item from the iterator. When exhausted, raise StopIteration' raise StopIteration def __iter__(self): return self @classmethod def __subclasshook__(cls, C): if cls is Iterator: if _hasattr(C, "next") and _hasattr(C, "__iter__"): return True return NotImplemented class Sized(object): __metaclass__ = ABCMeta @abstractmethod def __len__(self): return 0 @classmethod def __subclasshook__(cls, C): if cls is Sized: if _hasattr(C, "__len__"): return True return NotImplemented class Container(object): __metaclass__ = ABCMeta @abstractmethod def __contains__(self, x): return False @classmethod def __subclasshook__(cls, C): if cls is Container: if _hasattr(C, "__contains__"): return True return NotImplemented class Callable(object): __metaclass__ = ABCMeta @abstractmethod def __call__(self, *args, **kwds): return False @classmethod def __subclasshook__(cls, C): if cls is Callable: if _hasattr(C, "__call__"): return True return NotImplemented ### SETS ### class Set(Sized, Iterable, Container): """A set is a finite, iterable container. This class provides concrete generic implementations of all methods except for __contains__, __iter__ and __len__. To override the comparisons (presumably for speed, as the semantics are fixed), redefine __le__ and __ge__, then the other operations will automatically follow suit. """ def __le__(self, other): if not isinstance(other, Set): return NotImplemented if len(self) > len(other): return False for elem in self: if elem not in other: return False return True def __lt__(self, other): if not isinstance(other, Set): return NotImplemented return len(self) < len(other) and self.__le__(other) def __gt__(self, other): if not isinstance(other, Set): return NotImplemented return len(self) > len(other) and self.__ge__(other) def __ge__(self, other): if not isinstance(other, Set): return NotImplemented if len(self) < len(other): return False for elem in other: if elem not in self: return False return True def __eq__(self, other): if not isinstance(other, Set): return NotImplemented return len(self) == len(other) and self.__le__(other) def __ne__(self, other): return not (self == other) @classmethod def _from_iterable(cls, it): '''Construct an instance of the class from any iterable input. Must override this method if the class constructor signature does not accept an iterable for an input. ''' return cls(it) def __and__(self, other): if not isinstance(other, Iterable): return NotImplemented return self._from_iterable(value for value in other if value in self) __rand__ = __and__ def isdisjoint(self, other): 'Return True if two sets have a null intersection.' for value in other: if value in self: return False return True def __or__(self, other): if not isinstance(other, Iterable): return NotImplemented chain = (e for s in (self, other) for e in s) return self._from_iterable(chain) __ror__ = __or__ def __sub__(self, other): if not isinstance(other, Set): if not isinstance(other, Iterable): return NotImplemented other = self._from_iterable(other) return self._from_iterable(value for value in self if value not in other) def __rsub__(self, other): if not isinstance(other, Set): if not isinstance(other, Iterable): return NotImplemented other = self._from_iterable(other) return self._from_iterable(value for value in other if value not in self) def __xor__(self, other): if not isinstance(other, Set): if not isinstance(other, Iterable): return NotImplemented other = self._from_iterable(other) return (self - other) | (other - self) __rxor__ = __xor__ # Sets are not hashable by default, but subclasses can change this __hash__ = None def _hash(self): """Compute the hash value of a set. Note that we don't define __hash__: not all sets are hashable. But if you define a hashable set type, its __hash__ should call this function. This must be compatible __eq__. All sets ought to compare equal if they contain the same elements, regardless of how they are implemented, and regardless of the order of the elements; so there's not much freedom for __eq__ or __hash__. We match the algorithm used by the built-in frozenset type. """ MAX = sys.maxint MASK = 2 * MAX + 1 n = len(self) h = 1927868237 * (n + 1) h &= MASK for x in self: hx = hash(x) h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 h &= MASK h = h * 69069 + 907133923 h &= MASK if h > MAX: h -= MASK + 1 if h == -1: h = 590923713 return h Set.register(frozenset) class MutableSet(Set): """A mutable set is a finite, iterable container. This class provides concrete generic implementations of all methods except for __contains__, __iter__, __len__, add(), and discard(). To override the comparisons (presumably for speed, as the semantics are fixed), all you have to do is redefine __le__ and then the other operations will automatically follow suit. """ @abstractmethod def add(self, value): """Add an element.""" raise NotImplementedError @abstractmethod def discard(self, value): """Remove an element. Do not raise an exception if absent.""" raise NotImplementedError def remove(self, value): """Remove an element. If not a member, raise a KeyError.""" if value not in self: raise KeyError(value) self.discard(value) def pop(self): """Return the popped value. Raise KeyError if empty.""" it = iter(self) try: value = next(it) except StopIteration: raise KeyError self.discard(value) return value def clear(self): """This is slow (creates N new iterators!) but effective.""" try: while True: self.pop() except KeyError: pass def __ior__(self, it): for value in it: self.add(value) return self def __iand__(self, it): for value in (self - it): self.discard(value) return self def __ixor__(self, it): if it is self: self.clear() else: if not isinstance(it, Set): it = self._from_iterable(it) for value in it: if value in self: self.discard(value) else: self.add(value) return self def __isub__(self, it): if it is self: self.clear() else: for value in it: self.discard(value) return self MutableSet.register(set) ### MAPPINGS ### class Mapping(Sized, Iterable, Container): """A Mapping is a generic container for associating key/value pairs. This class provides concrete generic implementations of all methods except for __getitem__, __iter__, and __len__. """ @abstractmethod def __getitem__(self, key): raise KeyError def get(self, key, default=None): 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.' try: return self[key] except KeyError: return default def __contains__(self, key): try: self[key] except KeyError: return False else: return True def iterkeys(self): 'D.iterkeys() -> an iterator over the keys of D' return iter(self) def itervalues(self): 'D.itervalues() -> an iterator over the values of D' for key in self: yield self[key] def iteritems(self): 'D.iteritems() -> an iterator over the (key, value) items of D' for key in self: yield (key, self[key]) def keys(self): "D.keys() -> list of D's keys" return list(self) def items(self): "D.items() -> list of D's (key, value) pairs, as 2-tuples" return [(key, self[key]) for key in self] def values(self): "D.values() -> list of D's values" return [self[key] for key in self] # Mappings are not hashable by default, but subclasses can change this __hash__ = None def __eq__(self, other): if not isinstance(other, Mapping): return NotImplemented return dict(self.items()) == dict(other.items()) def __ne__(self, other): return not (self == other) class MappingView(Sized): def __init__(self, mapping): self._mapping = mapping def __len__(self): return len(self._mapping) def __repr__(self): #return '{0.__class__.__name__}({0._mapping!r})'.format(self) return '%s(%r)' % (self.__class__.__name__, self._mapping) #class KeysView(MappingView, Set): # # @classmethod # def _from_iterable(self, it): # return set(it) # # def __contains__(self, key): # return key in self._mapping # # def __iter__(self): # for key in self._mapping: # yield key # #KeysView.register(type({}.viewkeys())) # #class ItemsView(MappingView, Set): # # @classmethod # def _from_iterable(self, it): # return set(it) # # def __contains__(self, item): # key, value = item # try: # v = self._mapping[key] # except KeyError: # return False # else: # return v == value # # def __iter__(self): # for key in self._mapping: # yield (key, self._mapping[key]) # #ItemsView.register(type({}.viewitems())) # #class ValuesView(MappingView): # # def __contains__(self, value): # for key in self._mapping: # if value == self._mapping[key]: # return True # return False # # def __iter__(self): # for key in self._mapping: # yield self._mapping[key] # #ValuesView.register(type({}.viewvalues())) class MutableMapping(Mapping): """A MutableMapping is a generic container for associating key/value pairs. This class provides concrete generic implementations of all methods except for __getitem__, __setitem__, __delitem__, __iter__, and __len__. """ @abstractmethod def __setitem__(self, key, value): raise KeyError @abstractmethod def __delitem__(self, key): raise KeyError __marker = object() def pop(self, key, default=__marker): '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised. ''' try: value = self[key] except KeyError: if default is self.__marker: raise return default else: del self[key] return value def popitem(self): '''D.popitem() -> (k, v), remove and return some (key, value) pair as a 2-tuple; but raise KeyError if D is empty. ''' try: key = next(iter(self)) except StopIteration: raise KeyError value = self[key] del self[key] return key, value def clear(self): 'D.clear() -> None. Remove all items from D.' try: while True: self.popitem() except KeyError: pass def update(*args, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. If E present and has a .keys() method, does: for k in E: D[k] = E[k] If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v In either case, this is followed by: for k, v in F.items(): D[k] = v ''' if not args: raise TypeError("descriptor 'update' of 'MutableMapping' object " "needs an argument") self = args[0] args = args[1:] if len(args) > 1: raise TypeError('update expected at most 1 arguments, got %d' % len(args)) if args: other = args[0] if isinstance(other, Mapping): for key in other: self[key] = other[key] elif hasattr(other, "keys"): for key in other.keys(): self[key] = other[key] else: for key, value in other: self[key] = value for key, value in kwds.items(): self[key] = value def setdefault(self, key, default=None): 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D' try: return self[key] except KeyError: self[key] = default return default MutableMapping.register(dict) ### SEQUENCES ### class Sequence(Sized, Iterable, Container): """All the operations on a read-only sequence. Concrete subclasses must override __new__ or __init__, __getitem__, and __len__. """ @abstractmethod def __getitem__(self, index): raise IndexError def __iter__(self): i = 0 try: while True: v = self[i] yield v i += 1 except IndexError: return def __contains__(self, value): for v in self: if v == value: return True return False def __reversed__(self): for i in reversed(range(len(self))): yield self[i] def index(self, value): '''S.index(value) -> integer -- return first index of value. Raises ValueError if the value is not present. ''' for i, v in enumerate(self): if v == value: return i raise ValueError def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' return sum(1 for v in self if v == value) Sequence.register(tuple) Sequence.register(basestring) #Sequence.register(buffer) Sequence.register(xrange) class MutableSequence(Sequence): """All the operations on a read-only sequence. Concrete subclasses must provide __new__ or __init__, __getitem__, __setitem__, __delitem__, __len__, and insert(). """ @abstractmethod def __setitem__(self, index, value): raise IndexError @abstractmethod def __delitem__(self, index): raise IndexError @abstractmethod def insert(self, index, value): 'S.insert(index, object) -- insert object before index' raise IndexError def append(self, value): 'S.append(object) -- append object to the end of the sequence' self.insert(len(self), value) def reverse(self): 'S.reverse() -- reverse *IN PLACE*' n = len(self) for i in range(n//2): self[i], self[n-i-1] = self[n-i-1], self[i] def extend(self, values): 'S.extend(iterable) -- extend sequence by appending elements from the iterable' for v in values: self.append(v) def pop(self, index=-1): '''S.pop([index]) -> item -- remove and return item at index (default last). Raise IndexError if list is empty or index is out of range. ''' v = self[index] del self[index] return v def remove(self, value): '''S.remove(value) -- remove first occurrence of value. Raise ValueError if the value is not present. ''' del self[self.index(value)] def __iadd__(self, values): self.extend(values) return self MutableSequence.register(list) ================================================ FILE: third_party/stdlib/_weakrefset.py ================================================ # Access WeakSet through the weakref module. # This code is separated-out because it is needed # by abc.py to load everything else at startup. from '__go__/grumpy' import WeakRefType as ref __all__ = ['WeakSet'] class _IterationGuard(object): # This context manager registers itself in the current iterators of the # weak container, such as to delay all removals until the context manager # exits. # This technique should be relatively thread-safe (since sets are). def __init__(self, weakcontainer): # Don't create cycles self.weakcontainer = ref(weakcontainer) def __enter__(self): w = self.weakcontainer() if w is not None: w._iterating.add(self) return self def __exit__(self, e, t, b): w = self.weakcontainer() if w is not None: s = w._iterating s.remove(self) if not s: w._commit_removals() class WeakSet(object): def __init__(self, data=None): self.data = set() def _remove(item, selfref=ref(self)): self = selfref() if self is not None: if self._iterating: self._pending_removals.append(item) else: self.data.discard(item) self._remove = _remove # A list of keys to be removed self._pending_removals = [] self._iterating = set() if data is not None: self.update(data) def _commit_removals(self): l = self._pending_removals discard = self.data.discard while l: discard(l.pop()) def __iter__(self): with _IterationGuard(self): for itemref in self.data: item = itemref() if item is not None: # Caveat: the iterator will keep a strong reference to # `item` until it is resumed or closed. yield item def __len__(self): return len(self.data) - len(self._pending_removals) def __contains__(self, item): try: wr = ref(item) except TypeError: return False return wr in self.data def __reduce__(self): return (self.__class__, (list(self),), getattr(self, '__dict__', None)) __hash__ = None def add(self, item): if self._pending_removals: self._commit_removals() self.data.add(ref(item, self._remove)) def clear(self): if self._pending_removals: self._commit_removals() self.data.clear() def copy(self): return self.__class__(self) def pop(self): if self._pending_removals: self._commit_removals() while True: try: itemref = self.data.pop() except KeyError: raise KeyError('pop from empty WeakSet') item = itemref() if item is not None: return item def remove(self, item): if self._pending_removals: self._commit_removals() self.data.remove(ref(item)) def discard(self, item): if self._pending_removals: self._commit_removals() self.data.discard(ref(item)) def update(self, other): if self._pending_removals: self._commit_removals() for element in other: self.add(element) def __ior__(self, other): self.update(other) return self def difference(self, other): newset = self.copy() newset.difference_update(other) return newset __sub__ = difference def difference_update(self, other): self.__isub__(other) def __isub__(self, other): if self._pending_removals: self._commit_removals() if self is other: self.data.clear() else: self.data.difference_update(ref(item) for item in other) return self def intersection(self, other): return self.__class__(item for item in other if item in self) __and__ = intersection def intersection_update(self, other): self.__iand__(other) def __iand__(self, other): if self._pending_removals: self._commit_removals() self.data.intersection_update(ref(item) for item in other) return self def issubset(self, other): return self.data.issubset(ref(item) for item in other) __le__ = issubset def __lt__(self, other): return self.data < set(ref(item) for item in other) def issuperset(self, other): return self.data.issuperset(ref(item) for item in other) __ge__ = issuperset def __gt__(self, other): return self.data > set(ref(item) for item in other) def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented return self.data == set(ref(item) for item in other) def __ne__(self, other): opposite = self.__eq__(other) if opposite is NotImplemented: return NotImplemented return not opposite def symmetric_difference(self, other): newset = self.copy() newset.symmetric_difference_update(other) return newset __xor__ = symmetric_difference def symmetric_difference_update(self, other): self.__ixor__(other) def __ixor__(self, other): if self._pending_removals: self._commit_removals() if self is other: self.data.clear() else: self.data.symmetric_difference_update(ref(item, self._remove) for item in other) return self def union(self, other): return self.__class__(e for s in (self, other) for e in s) __or__ = union def isdisjoint(self, other): return len(self.intersection(other)) == 0 ================================================ FILE: third_party/stdlib/abc.py ================================================ # Copyright 2007 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Abstract Base Classes (ABCs) according to PEP 3119.""" import _weakrefset import types # Instance of old-style class # class _C(object): pass # _InstanceType = type(_C()) def abstractmethod(funcobj): """A decorator indicating abstract methods. Requires that the metaclass is ABCMeta or derived from it. A class that has a metaclass derived from ABCMeta cannot be instantiated unless all of its abstract methods are overridden. The abstract methods can be called using any of the normal 'super' call mechanisms. Usage: class C: __metaclass__ = ABCMeta @abstractmethod def my_abstract_method(self, ...): ... """ funcobj.__isabstractmethod__ = True return funcobj class abstractproperty(property): """A decorator indicating abstract properties. Requires that the metaclass is ABCMeta or derived from it. A class that has a metaclass derived from ABCMeta cannot be instantiated unless all of its abstract properties are overridden. The abstract properties can be called using any of the normal 'super' call mechanisms. Usage: class C: __metaclass__ = ABCMeta @abstractproperty def my_abstract_property(self): ... This defines a read-only property; you can also define a read-write abstract property using the 'long' form of property declaration: class C: __metaclass__ = ABCMeta def getx(self): ... def setx(self, value): ... x = abstractproperty(getx, setx) """ __isabstractmethod__ = True class ABCMeta(type): """Metaclass for defining Abstract Base Classes (ABCs). Use this metaclass to create an ABC. An ABC can be subclassed directly, and then acts as a mix-in class. You can also register unrelated concrete classes (even built-in classes) and unrelated ABCs as 'virtual subclasses' -- these and their descendants will be considered subclasses of the registering ABC by the built-in issubclass() function, but the registering ABC won't show up in their MRO (Method Resolution Order) nor will method implementations defined by the registering ABC be callable (not even via super()). """ # A global counter that is incremented each time a class is # registered as a virtual subclass of anything. It forces the # negative cache to be cleared before its next use. _abc_invalidation_counter = 0 def __new__(mcls, name, bases, namespace): cls = super(ABCMeta, mcls).__new__(mcls, name, bases, namespace) # Compute set of abstract method names abstracts = set(name for name, value in namespace.items() if getattr(value, "__isabstractmethod__", False)) for base in bases: for name in getattr(base, "__abstractmethods__", set()): value = getattr(cls, name, None) if getattr(value, "__isabstractmethod__", False): abstracts.add(name) cls.__abstractmethods__ = frozenset(abstracts) # Set up inheritance registry cls._abc_registry = _weakrefset.WeakSet() cls._abc_cache = _weakrefset.WeakSet() cls._abc_negative_cache = _weakrefset.WeakSet() cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter return cls def register(cls, subclass): """Register a virtual subclass of an ABC.""" if not isinstance(subclass, type): raise TypeError("Can only register classes") if issubclass(subclass, cls): return # Already a subclass # Subtle: test for cycles *after* testing for "already a subclass"; # this means we allow X.register(X) and interpret it as a no-op. if issubclass(cls, subclass): # This would create a cycle, which is bad for the algorithm below raise RuntimeError("Refusing to create an inheritance cycle") cls._abc_registry.add(subclass) ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache def _dump_registry(cls, file=None): """Debug helper to print the ABC registry.""" print >> file, "Class: %s.%s" % (cls.__module__, cls.__name__) print >> file, "Inv.counter: %s" % ABCMeta._abc_invalidation_counter for name in sorted(cls.__dict__.keys()): if name.startswith("_abc_"): value = getattr(cls, name) print >> file, "%s: %r" % (name, value) def __instancecheck__(cls, instance): """Override for isinstance(instance, cls).""" # Inline the cache checking when it's simple. subclass = getattr(instance, '__class__', None) if subclass is not None and subclass in cls._abc_cache: return True subtype = type(instance) # Old-style instances # if subtype is _InstanceType: # subtype = subclass if subtype is subclass or subclass is None: if (cls._abc_negative_cache_version == ABCMeta._abc_invalidation_counter and subtype in cls._abc_negative_cache): return False # Fall back to the subclass check. return cls.__subclasscheck__(subtype) return (cls.__subclasscheck__(subclass) or cls.__subclasscheck__(subtype)) def __subclasscheck__(cls, subclass): """Override for issubclass(subclass, cls).""" # Check cache if subclass in cls._abc_cache: return True # Check negative cache; may have to invalidate if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: # Invalidate the negative cache cls._abc_negative_cache = _weakrefset.WeakSet() cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter elif subclass in cls._abc_negative_cache: return False # Check the subclass hook ok = cls.__subclasshook__(subclass) if ok is not NotImplemented: assert isinstance(ok, bool) if ok: cls._abc_cache.add(subclass) else: cls._abc_negative_cache.add(subclass) return ok # Check if it's a direct subclass if cls in getattr(subclass, '__mro__', ()): cls._abc_cache.add(subclass) return True # Check if it's a subclass of a registered class (recursive) for rcls in cls._abc_registry: if issubclass(subclass, rcls): cls._abc_cache.add(subclass) return True # Check if it's a subclass of a subclass (recursive) for scls in cls.__subclasses__(): if issubclass(subclass, scls): cls._abc_cache.add(subclass) return True # No dice; update negative cache cls._abc_negative_cache.add(subclass) return False ================================================ FILE: third_party/stdlib/argparse.py ================================================ # Author: Steven J. Bethard . """Command-line parsing library This module is an optparse-inspired command-line parsing library that: - handles both optional and positional arguments - produces highly informative usage messages - supports parsers that dispatch to sub-parsers The following is a simple usage example that sums integers from the command-line and writes the result to a file:: parser = argparse.ArgumentParser( description='sum the integers at the command line') parser.add_argument( 'integers', metavar='int', nargs='+', type=int, help='an integer to be summed') parser.add_argument( '--log', default=sys.stdout, type=argparse.FileType('w'), help='the file where the sum should be written') args = parser.parse_args() args.log.write('%s' % sum(args.integers)) args.log.close() The module contains the following public classes: - ArgumentParser -- The main entry point for command-line parsing. As the example above shows, the add_argument() method is used to populate the parser with actions for optional and positional arguments. Then the parse_args() method is invoked to convert the args at the command-line into an object with attributes. - ArgumentError -- The exception raised by ArgumentParser objects when there are errors with the parser's actions. Errors raised while parsing the command-line are caught by ArgumentParser and emitted as command-line messages. - FileType -- A factory for defining types of files to be created. As the example above shows, instances of FileType are typically passed as the type= argument of add_argument() calls. - Action -- The base class for parser actions. Typically actions are selected by passing strings like 'store_true' or 'append_const' to the action= argument of add_argument(). However, for greater customization of ArgumentParser actions, subclasses of Action may be defined and passed as the action= argument. - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, ArgumentDefaultsHelpFormatter -- Formatter classes which may be passed as the formatter_class= argument to the ArgumentParser constructor. HelpFormatter is the default, RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser not to change the formatting for help text, and ArgumentDefaultsHelpFormatter adds information about argument defaults to the help. All other classes in this module are considered implementation details. (Also note that HelpFormatter and RawDescriptionHelpFormatter are only considered public as object names -- the API of the formatter objects is still considered an implementation detail.) """ __version__ = '1.1' __all__ = [ 'ArgumentParser', 'ArgumentError', 'ArgumentTypeError', 'FileType', 'HelpFormatter', 'ArgumentDefaultsHelpFormatter', 'RawDescriptionHelpFormatter', 'RawTextHelpFormatter', 'Namespace', 'Action', 'ONE_OR_MORE', 'OPTIONAL', 'PARSER', 'REMAINDER', 'SUPPRESS', 'ZERO_OR_MORE', ] import collections as _collections # import copy as _copy import os as _os import re as _re import sys as _sys import textwrap as _textwrap # from gettext import gettext as _ _ = lambda x: x def setdefault(d, k, default=None): if k not in d: d[k] = default return d[k] def dict_pop(d, k, default=None): if k in d: ret = d[k] del d[k] return ret if default: return default def list_remove(l, x): for i in range(len(l)): if l[i] == x: l.pop(i) break MAPPING_SUB = _re.compile(r'%\(([^)]+)\)s').sub def _callable(obj): return hasattr(obj, '__call__') or hasattr(obj, '__bases__') SUPPRESS = '==SUPPRESS==' OPTIONAL = '?' ZERO_OR_MORE = '*' ONE_OR_MORE = '+' PARSER = 'A...' REMAINDER = '...' _UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' # ============================= # Utility functions and classes # ============================= class _AttributeHolder(object): """Abstract base class that provides __repr__. The __repr__ method returns a string in the format:: ClassName(attr=name, attr=name, ...) The attributes are determined either by a class-level attribute, '_kwarg_names', or by inspecting the instance __dict__. """ def __repr__(self): type_name = type(self).__name__ arg_strings = [] for arg in self._get_args(): arg_strings.append(repr(arg)) for name, value in self._get_kwargs(): arg_strings.append('%s=%r' % (name, value)) return '%s(%s)' % (type_name, ', '.join(arg_strings)) def _get_kwargs(self): return sorted(self.__dict__.items()) def _get_args(self): return [] def _ensure_value(namespace, name, value): if getattr(namespace, name, None) is None: setattr(namespace, name, value) return getattr(namespace, name) # =============== # Formatting Help # =============== class HelpFormatter(object): """Formatter for generating usage messages and argument help strings. Only the name of this class is considered a public API. All the methods provided by the class are considered an implementation detail. """ def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): # default setting for width if width is None: try: width = int(_os.environ['COLUMNS']) except (KeyError, ValueError): width = 80 width -= 2 self._prog = prog self._indent_increment = indent_increment self._max_help_position = max_help_position self._max_help_position = min(max_help_position, max(width - 20, indent_increment * 2)) self._width = width self._current_indent = 0 self._level = 0 self._action_max_length = 0 self._root_section = self._Section(self, None) self._current_section = self._root_section self._whitespace_matcher = _re.compile(r'\s+') self._long_break_matcher = _re.compile(r'\n\n\n+') # =============================== # Section and indentation methods # =============================== def _indent(self): self._current_indent += self._indent_increment self._level += 1 def _dedent(self): self._current_indent -= self._indent_increment assert self._current_indent >= 0, 'Indent decreased below 0.' self._level -= 1 class _Section(object): def __init__(self, formatter, parent, heading=None): self.formatter = formatter self.parent = parent self.heading = heading self.items = [] def format_help(self): # format the indented section if self.parent is not None: self.formatter._indent() join = self.formatter._join_parts for func, args in self.items: func(*args) item_help = join([func(*args) for func, args in self.items]) if self.parent is not None: self.formatter._dedent() # return nothing if the section was empty if not item_help: return '' # add the heading if the section was non-empty if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent # heading = '%*s%s:\n' % (current_indent, '', self.heading) heading = ' ' * current_indent + self.heading + ':\n' else: heading = '' # join the section-initial newline, the heading and the help return join(['\n', heading, item_help, '\n']) def _add_item(self, func, args): self._current_section.items.append((func, args)) # ======================== # Message building methods # ======================== def start_section(self, heading): self._indent() section = self._Section(self, self._current_section, heading) self._add_item(section.format_help, []) self._current_section = section def end_section(self): self._current_section = self._current_section.parent self._dedent() def add_text(self, text): if text is not SUPPRESS and text is not None: self._add_item(self._format_text, [text]) def add_usage(self, usage, actions, groups, prefix=None): if usage is not SUPPRESS: args = usage, actions, groups, prefix self._add_item(self._format_usage, args) def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations get_invocation = self._format_action_invocation invocations = [get_invocation(action)] for subaction in self._iter_indented_subactions(action): invocations.append(get_invocation(subaction)) # update the maximum item length invocation_length = max([len(s) for s in invocations]) action_length = invocation_length + self._current_indent self._action_max_length = max(self._action_max_length, action_length) # add the item to the list self._add_item(self._format_action, [action]) def add_arguments(self, actions): for action in actions: self.add_argument(action) # ======================= # Help-formatting methods # ======================= def format_help(self): help = self._root_section.format_help() if help: help = self._long_break_matcher.sub('\n\n', help) help = help.strip('\n') + '\n' return help def _join_parts(self, part_strings): return ''.join([part for part in part_strings if part and part is not SUPPRESS]) def _format_usage(self, usage, actions, groups, prefix): if prefix is None: prefix = _('usage: ') # if usage is specified, use that if usage is not None: # usage = usage % dict(prog=self._prog) usage = usage.replace('%(prog)s', str(self._prog)) # if no optionals or positionals are available, usage is just prog elif usage is None and not actions: # usage = '%(prog)s' % dict(prog=self._prog) usage = self._prog # if optionals and positionals are available, calculate usage elif usage is None: # prog = '%(prog)s' % dict(prog=self._prog) prog = self._prog # split optionals from positionals optionals = [] positionals = [] for action in actions: if action.option_strings: optionals.append(action) else: positionals.append(action) # build full usage string format = self._format_actions_usage action_usage = format(optionals + positionals, groups) usage = ' '.join([s for s in [prog, action_usage] if s]) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(usage) > text_width: # break usage into wrappable parts part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' opt_usage = format(optionals, groups) pos_usage = format(positionals, groups) opt_parts = _re.findall(part_regexp, opt_usage) pos_parts = _re.findall(part_regexp, pos_usage) assert ' '.join(opt_parts) == opt_usage assert ' '.join(pos_parts) == pos_usage # helper for wrapping lines def get_lines(parts, indent, prefix=None): lines = [] line = [] if prefix is not None: line_len = len(prefix) - 1 else: line_len = len(indent) - 1 for part in parts: if line_len + 1 + len(part) > text_width and line: lines.append(indent + ' '.join(line)) line = [] line_len = len(indent) - 1 line.append(part) line_len += len(part) + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: lines[0] = lines[0][len(indent):] return lines # if prog is short, follow it with optionals or positionals if len(prefix) + len(prog) <= 0.75 * text_width: indent = ' ' * (len(prefix) + len(prog) + 1) if opt_parts: lines = get_lines([prog] + opt_parts, indent, prefix) # lines.extend(get_lines(pos_parts, indent)) lines += (get_lines(pos_parts, indent)) elif pos_parts: lines = get_lines([prog] + pos_parts, indent, prefix) else: lines = [prog] # if prog is long, put it on its own line else: indent = ' ' * len(prefix) parts = opt_parts + pos_parts lines = get_lines(parts, indent) if len(lines) > 1: lines = [] # lines.extend(get_lines(opt_parts, indent)) lines += (get_lines(opt_parts, indent)) # lines.extend(get_lines(pos_parts, indent)) lines += (get_lines(pos_parts, indent)) lines = [prog] + lines # join lines into usage usage = '\n'.join(lines) # prefix with 'usage:' return '%s%s\n\n' % (prefix, usage) def _format_actions_usage(self, actions, groups): # find group indices and identify actions in groups group_actions = set() inserts = {} for group in groups: try: # start = actions.index(group._group_actions[0]) start = None for i in range(len(actions)): if actions[i] == group._group_actions[0]: start = i break if start is None: raise ValueError except ValueError: continue else: end = start + len(group._group_actions) if actions[start:end] == group._group_actions: for action in group._group_actions: group_actions.add(action) if not group.required: if start in inserts: inserts[start] += ' [' else: inserts[start] = '[' inserts[end] = ']' else: if start in inserts: inserts[start] += ' (' else: inserts[start] = '(' inserts[end] = ')' for i in range(start + 1, end): inserts[i] = '|' # collect all actions format strings parts = [] for i, action in enumerate(actions): # suppressed arguments are marked with None # remove | separators for suppressed arguments if action.help is SUPPRESS: parts.append(None) if inserts.get(i) == '|': # inserts.pop(i) if i in inserts: del inserts[i] elif inserts.get(i + 1) == '|': # inserts.pop(i + 1) if i + 1 in inserts: del inserts[i + 1] # produce all arg strings elif not action.option_strings: part = self._format_args(action, action.dest) # if it's in a group, strip the outer [] if action in group_actions: if part[0] == '[' and part[-1] == ']': part = part[1:-1] # add the action string to the list parts.append(part) # produce the first way to invoke the option in brackets else: option_string = action.option_strings[0] # if the Optional doesn't take a value, format is: # -s or --long if action.nargs == 0: part = '%s' % option_string # if the Optional takes a value, format is: # -s ARGS or --long ARGS else: default = action.dest.upper() args_string = self._format_args(action, default) part = '%s %s' % (option_string, args_string) # make it look optional if it's not required or in a group if not action.required and action not in group_actions: part = '[%s]' % part # add the action string to the list parts.append(part) # insert things at the necessary indices for i in sorted(inserts, reverse=True): parts[i:i] = [inserts[i]] # join all the action items with spaces text = ' '.join([item for item in parts if item is not None]) # clean up separators for mutually exclusive groups open_bracket = r'[\[(]' close = r'[\])]' # text = _re.sub(r'(%s) ' % open_bracket, r'\1', text) text = text.replace('[ ', '[').replace('( ', '(') # text = _re.sub(r' (%s)' % close, r'\1', text) text = text.replace(' ]', ']').replace(' )', ')') text = _re.sub(r'%s *%s' % (open_bracket, close), r'', text) # text = _re.sub(r'\(([^|]*)\)', r'\1', text) text = _re.sub(r'\(([^|]*)\)', lambda x: x.group(1), text) text = text.strip() # return the text return text def _format_text(self, text): if '%(prog)s' in text: # text = text % dict(prog=self._prog) text = text.replace('%(prog)s', self._prog) text_width = max(self._width - self._current_indent, 11) indent = ' ' * self._current_indent return self._fill_text(text, text_width, indent) + '\n\n' def _format_action(self, action): # determine the required width and the entry label help_position = min(self._action_max_length + 2, self._max_help_position) help_width = max(self._width - help_position, 11) action_width = help_position - self._current_indent - 2 action_header = self._format_action_invocation(action) # ho nelp; start on same line and add a final newline if not action.help: tup = self._current_indent, '', action_header # action_header = '%*s%s\n' % tup action_header = ' ' * self._current_indent + action_header + '\n' # short action name; start on the same line and pad two spaces elif len(action_header) <= action_width: tup = self._current_indent, '', action_width, action_header # action_header = '%*s%-*s ' % tup action_header = ' ' * self._current_indent + (action_header + ' ' * action_width)[:action_width] + ' ' indent_first = 0 # long action name; start on the next line else: tup = self._current_indent, '', action_header # action_header = '%*s%s\n' % tup action_header = ' ' * self._current_indent + action_header + '\n' indent_first = help_position # collect the pieces of the action help parts = [action_header] # if there was help for the action, add lines of help text if action.help: help_text = self._expand_help(action) help_lines = self._split_lines(help_text, help_width) # parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) parts.append(' ' * indent_first + help_lines[0] + '\n') for line in help_lines[1:]: # parts.append('%*s%s\n' % (help_position, '', line)) parts.append(' ' * help_position + line + '\n') # or add a newline if the description doesn't end with one elif not action_header.endswith('\n'): parts.append('\n') # if there are any sub-actions, add their help as well for subaction in self._iter_indented_subactions(action): parts.append(self._format_action(subaction)) # return a single string return self._join_parts(parts) def _format_action_invocation(self, action): if not action.option_strings: metavar, = self._metavar_formatter(action, action.dest)(1) return metavar else: parts = [] # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: # parts.extend(action.option_strings) parts += (action.option_strings) # if the Optional takes a value, format is: # -s ARGS, --long ARGS else: default = action.dest.upper() args_string = self._format_args(action, default) for option_string in action.option_strings: parts.append('%s %s' % (option_string, args_string)) return ', '.join(parts) def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: result = action.metavar elif action.choices is not None: choice_strs = [str(choice) for choice in action.choices] # result = '{%s}' % ','.join(choice_strs) result = '{%s}' % ','.join(sorted(choice_strs)) else: result = default_metavar def format(tuple_size): if isinstance(result, tuple): return result else: return (result, ) * tuple_size return format def _format_args(self, action, default_metavar): get_metavar = self._metavar_formatter(action, default_metavar) if action.nargs is None: result = '%s' % get_metavar(1) elif action.nargs == OPTIONAL: result = '[%s]' % get_metavar(1) elif action.nargs == ZERO_OR_MORE: result = '[%s [%s ...]]' % get_metavar(2) elif action.nargs == ONE_OR_MORE: result = '%s [%s ...]' % get_metavar(2) elif action.nargs == REMAINDER: result = '...' elif action.nargs == PARSER: result = '%s ...' % get_metavar(1) else: formats = ['%s' for _ in range(action.nargs)] result = ' '.join(formats) % get_metavar(action.nargs) return result def _expand_help(self, action): # params = dict(vars(action), prog=self._prog) params = dict(action.__dict__, prog=self._prog) for name in list(params): if params[name] is SUPPRESS: del params[name] for name in list(params): if hasattr(params[name], '__name__'): params[name] = params[name].__name__ if params.get('choices') is not None: choices_str = ', '.join([str(c) for c in params['choices']]) params['choices'] = choices_str # return self._get_help_string(action) % params return MAPPING_SUB(lambda x: str(params.get( x.group(1), x.group(0))), self._get_help_string(action)).replace('%%', '%') def _iter_indented_subactions(self, action): try: get_subactions = action._get_subactions except AttributeError: pass else: self._indent() for subaction in get_subactions(): yield subaction self._dedent() def _split_lines(self, text, width): text = self._whitespace_matcher.sub(' ', text).strip() return _textwrap.wrap(text, width) def _fill_text(self, text, width, indent): text = self._whitespace_matcher.sub(' ', text).strip() return _textwrap.fill(text, width, initial_indent=indent, subsequent_indent=indent) def _get_help_string(self, action): return action.help class RawDescriptionHelpFormatter(HelpFormatter): """Help message formatter which retains any formatting in descriptions. Only the name of this class is considered a public API. All the methods provided by the class are considered an implementation detail. """ def _fill_text(self, text, width, indent): return ''.join([indent + line for line in text.splitlines(True)]) class RawTextHelpFormatter(RawDescriptionHelpFormatter): """Help message formatter which retains formatting of all help text. Only the name of this class is considered a public API. All the methods provided by the class are considered an implementation detail. """ def _split_lines(self, text, width): return text.splitlines() class ArgumentDefaultsHelpFormatter(HelpFormatter): """Help message formatter which adds default values to argument help. Only the name of this class is considered a public API. All the methods provided by the class are considered an implementation detail. """ def _get_help_string(self, action): help = action.help if '%(default)' not in action.help: if action.default is not SUPPRESS: defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] if action.option_strings or action.nargs in defaulting_nargs: help += ' (default: %(default)s)' return help # ===================== # Options and Arguments # ===================== def _get_action_name(argument): if argument is None: return None elif argument.option_strings: return '/'.join(argument.option_strings) elif argument.metavar not in (None, SUPPRESS): return argument.metavar elif argument.dest not in (None, SUPPRESS): return argument.dest else: return None class ArgumentError(Exception): """An error from creating or using an argument (optional or positional). The string value of this exception is the message, augmented with information about the argument that caused it. """ def __init__(self, argument, message): self.argument_name = _get_action_name(argument) self.message = message def __str__(self): if self.argument_name is None: format = '%(message)s' else: format = 'argument %(argument_name)s: %(message)s' # return format % dict(message=self.message, # argument_name=self.argument_name) return format.replace('%(message)s', str(self.message)).replace('%(argument_name)s', str(self.argument_name)) class ArgumentTypeError(Exception): """An error from trying to convert a command line string to a type.""" pass # ============== # Action classes # ============== class Action(_AttributeHolder): """Information about how to convert command line strings to Python objects. Action objects are used by an ArgumentParser to represent the information needed to parse a single argument from one or more strings from the command line. The keyword arguments to the Action constructor are also all attributes of Action instances. Keyword Arguments: - option_strings -- A list of command-line option strings which should be associated with this action. - dest -- The name of the attribute to hold the created object(s) - nargs -- The number of command-line arguments that should be consumed. By default, one argument will be consumed and a single value will be produced. Other values include: - N (an integer) consumes N arguments (and produces a list) - '?' consumes zero or one arguments - '*' consumes zero or more arguments (and produces a list) - '+' consumes one or more arguments (and produces a list) Note that the difference between the default and nargs=1 is that with the default, a single value will be produced, while with nargs=1, a list containing a single value will be produced. - const -- The value to be produced if the option is specified and the option uses an action that takes no values. - default -- The value to be produced if the option is not specified. - type -- A callable that accepts a single string argument, and returns the converted value. The standard Python types str, int, float, and complex are useful examples of such callables. If None, str is used. - choices -- A container of values that should be allowed. If not None, after a command-line argument has been converted to the appropriate type, an exception will be raised if it is not a member of this collection. - required -- True if the action must always be specified at the command line. This is only meaningful for optional command-line arguments. - help -- The help string describing the argument. - metavar -- The name to be used for the option's argument with the help string. If None, the 'dest' value will be used as the name. """ def __init__(self, option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None): self.option_strings = option_strings self.dest = dest self.nargs = nargs self.const = const self.default = default self.type = type self.choices = choices self.required = required self.help = help self.metavar = metavar def _get_kwargs(self): names = [ 'option_strings', 'dest', 'nargs', 'const', 'default', 'type', 'choices', 'help', 'metavar', ] return [(name, getattr(self, name)) for name in names] def __call__(self, parser, namespace, values, option_string=None): raise NotImplementedError(_('.__call__() not defined')) class _StoreAction(Action): def __init__(self, option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None): if nargs == 0: raise ValueError('nargs for store actions must be > 0; if you ' 'have nothing to store, actions such as store ' 'true or store const may be more appropriate') if const is not None and nargs != OPTIONAL: raise ValueError('nargs must be %r to supply const' % OPTIONAL) super(_StoreAction, self).__init__( option_strings=option_strings, dest=dest, nargs=nargs, const=const, default=default, type=type, choices=choices, required=required, help=help, metavar=metavar) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) class _StoreConstAction(Action): def __init__(self, option_strings, dest, const, default=None, required=False, help=None, metavar=None): super(_StoreConstAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, const=const, default=default, required=required, help=help) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, self.const) class _StoreTrueAction(_StoreConstAction): def __init__(self, option_strings, dest, default=False, required=False, help=None): super(_StoreTrueAction, self).__init__( option_strings=option_strings, dest=dest, const=True, default=default, required=required, help=help) class _StoreFalseAction(_StoreConstAction): def __init__(self, option_strings, dest, default=True, required=False, help=None): super(_StoreFalseAction, self).__init__( option_strings=option_strings, dest=dest, const=False, default=default, required=required, help=help) class _AppendAction(Action): def __init__(self, option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None): if nargs == 0: raise ValueError('nargs for append actions must be > 0; if arg ' 'strings are not supplying the value to append, ' 'the append const action may be more appropriate') if const is not None and nargs != OPTIONAL: raise ValueError('nargs must be %r to supply const' % OPTIONAL) super(_AppendAction, self).__init__( option_strings=option_strings, dest=dest, nargs=nargs, const=const, default=default, type=type, choices=choices, required=required, help=help, metavar=metavar) def __call__(self, parser, namespace, values, option_string=None): # items = _copy.copy(_ensure_value(namespace, self.dest, [])) items = (_ensure_value(namespace, self.dest, []))[:] items.append(values) setattr(namespace, self.dest, items) class _AppendConstAction(Action): def __init__(self, option_strings, dest, const, default=None, required=False, help=None, metavar=None): super(_AppendConstAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, const=const, default=default, required=required, help=help, metavar=metavar) def __call__(self, parser, namespace, values, option_string=None): # items = _copy.copy(_ensure_value(namespace, self.dest, [])) items = (_ensure_value(namespace, self.dest, []))[:] items.append(self.const) setattr(namespace, self.dest, items) class _CountAction(Action): def __init__(self, option_strings, dest, default=None, required=False, help=None): super(_CountAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, default=default, required=required, help=help) def __call__(self, parser, namespace, values, option_string=None): new_count = _ensure_value(namespace, self.dest, 0) + 1 setattr(namespace, self.dest, new_count) class _HelpAction(Action): def __init__(self, option_strings, dest=SUPPRESS, default=SUPPRESS, help=None): super(_HelpAction, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, help=help) def __call__(self, parser, namespace, values, option_string=None): parser.print_help() parser.exit() class _VersionAction(Action): def __init__(self, option_strings, version=None, dest=SUPPRESS, default=SUPPRESS, help="show program's version number and exit"): super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, help=help) self.version = version def __call__(self, parser, namespace, values, option_string=None): version = self.version if version is None: version = parser.version formatter = parser._get_formatter() formatter.add_text(version) parser.exit(message=formatter.format_help()) class _SubParsersAction(Action): class _ChoicesPseudoAction(Action): def __init__(self, name, help): sup = super(_SubParsersAction._ChoicesPseudoAction, self) sup.__init__(option_strings=[], dest=name, help=help) def __init__(self, option_strings, prog, parser_class, dest=SUPPRESS, help=None, metavar=None): self._prog_prefix = prog self._parser_class = parser_class self._name_parser_map = {} # _collections.OrderedDict() self._choices_actions = [] super(_SubParsersAction, self).__init__( option_strings=option_strings, dest=dest, nargs=PARSER, choices=self._name_parser_map, help=help, metavar=metavar) def add_parser(self, name, **kwargs): # set prog from the existing prefix if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) # create a pseudo-action to hold the choice help if 'help' in kwargs: # help = kwargs.pop('help') help = kwargs['help'] del kwargs['help'] choice_action = self._ChoicesPseudoAction(name, help) self._choices_actions.append(choice_action) # create the parser and add it to the map parser = self._parser_class(**kwargs) self._name_parser_map[name] = parser return parser def _get_subactions(self): return self._choices_actions def __call__(self, parser, namespace, values, option_string=None): parser_name = values[0] arg_strings = values[1:] # set the parser name if requested if self.dest is not SUPPRESS: setattr(namespace, self.dest, parser_name) # select the parser try: parser = self._name_parser_map[parser_name] except KeyError: tup = parser_name, ', '.join(self._name_parser_map) msg = _('unknown parser %r (choices: %s)') % tup raise ArgumentError(self, msg) # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them # In case this subparser defines new defaults, we parse them # in a new namespace object and then update the original # namespace for the relevant parts. subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) # for key, value in vars(subnamespace).items(): for key, value in subnamespace.__dict__.items(): setattr(namespace, key, value) if arg_strings: # vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) args_attr = setdefault(namespace.__dict__, _UNRECOGNIZED_ARGS_ATTR, []) # getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) args_attr += (arg_strings) # ============== # Type classes # ============== class FileType(object): """Factory for creating file object types Instances of FileType are typically passed as type= arguments to the ArgumentParser add_argument() method. Keyword Arguments: - mode -- A string indicating how the file is to be opened. Accepts the same values as the builtin open() function. - bufsize -- The file's desired buffer size. Accepts the same values as the builtin open() function. """ def __init__(self, mode='r', bufsize=-1): self._mode = mode self._bufsize = bufsize def __call__(self, string): # the special argument "-" means sys.std{in,out} if string == '-': if 'r' in self._mode: return _sys.stdin elif 'w' in self._mode: return _sys.stdout else: msg = _('argument "-" with mode %r') % self._mode raise ValueError(msg) # all other arguments are used as file names try: return open(string, self._mode, self._bufsize) except IOError as e: message = _("can't open '%s': %s") raise ArgumentTypeError(message % (string, e)) def __repr__(self): args = self._mode, self._bufsize args_str = ', '.join(repr(arg) for arg in args if arg != -1) return '%s(%s)' % (type(self).__name__, args_str) # =========================== # Optional and Positional Parsing # =========================== class Namespace(_AttributeHolder): """Simple object for storing attributes. Implements equality by attribute names and values, and provides a simple string representation. """ def __init__(self, **kwargs): for name in kwargs: setattr(self, name, kwargs[name]) __hash__ = None def __eq__(self, other): if not isinstance(other, Namespace): return NotImplemented # return vars(self) == vars(other) return self.__dict__ == other.__dict__ def __ne__(self, other): if not isinstance(other, Namespace): return NotImplemented return not (self == other) def __contains__(self, key): return key in self.__dict__ class _ActionsContainer(object): def __init__(self, description, prefix_chars, argument_default, conflict_handler): # super(_ActionsContainer, self).__init__() self.description = description self.argument_default = argument_default self.prefix_chars = prefix_chars self.conflict_handler = conflict_handler # set up registries self._registries = {} # register actions self.register('action', None, _StoreAction) self.register('action', 'store', _StoreAction) self.register('action', 'store_const', _StoreConstAction) self.register('action', 'store_true', _StoreTrueAction) self.register('action', 'store_false', _StoreFalseAction) self.register('action', 'append', _AppendAction) self.register('action', 'append_const', _AppendConstAction) self.register('action', 'count', _CountAction) self.register('action', 'help', _HelpAction) self.register('action', 'version', _VersionAction) self.register('action', 'parsers', _SubParsersAction) # raise an exception if the conflict handler is invalid self._get_handler() # action storage self._actions = [] self._option_string_actions = {} # groups self._action_groups = [] self._mutually_exclusive_groups = [] # defaults storage self._defaults = {} # determines whether an "option" looks like a negative number self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited self._has_negative_number_optionals = [] # ==================== # Registration methods # ==================== def register(self, registry_name, value, object): # registry = self._registries.setdefault(registry_name, {}) registry = setdefault(self._registries, registry_name, {}) registry[value] = object def _registry_get(self, registry_name, value, default=None): return self._registries[registry_name].get(value, default) # ================================== # Namespace default accessor methods # ================================== def set_defaults(self, **kwargs): self._defaults.update(kwargs) # if these defaults match any existing arguments, replace # the previous default on the object with the new one for action in self._actions: if action.dest in kwargs: action.default = kwargs[action.dest] def get_default(self, dest): for action in self._actions: if action.dest == dest and action.default is not None: return action.default return self._defaults.get(dest, None) # ======================= # Adding argument actions # ======================= def add_argument(self, *args, **kwargs): """ add_argument(dest, ..., name=value, ...) add_argument(option_string, option_string, ..., name=value, ...) """ # if no positional args are supplied or only one is supplied and # it doesn't look like an option string, parse a positional # argument chars = self.prefix_chars if not args or len(args) == 1 and args[0][0] not in chars: if args and 'dest' in kwargs: raise ValueError('dest supplied twice for positional argument') kwargs = self._get_positional_kwargs(*args, **kwargs) # otherwise, we're adding an optional argument else: kwargs = self._get_optional_kwargs(*args, **kwargs) # if no default was supplied, use the parser-level default if 'default' not in kwargs: dest = kwargs['dest'] if dest in self._defaults: kwargs['default'] = self._defaults[dest] elif self.argument_default is not None: kwargs['default'] = self.argument_default # create the action object, and add it to the parser action_class = self._pop_action_class(kwargs) if not _callable(action_class): raise ValueError('unknown action "%s"' % (action_class,)) action = action_class(**kwargs) # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not _callable(type_func): raise ValueError('%r is not callable' % (type_func,)) # raise an error if the metavar does not match the type if hasattr(self, "_get_formatter"): try: self._get_formatter()._format_args(action, None) except TypeError: raise ValueError("length of metavar tuple does not match nargs") return self._add_action(action) def add_argument_group(self, *args, **kwargs): group = _ArgumentGroup(self, *args, **kwargs) self._action_groups.append(group) return group def add_mutually_exclusive_group(self, **kwargs): group = _MutuallyExclusiveGroup(self, **kwargs) self._mutually_exclusive_groups.append(group) return group def _add_action(self, action): # resolve any conflicts self._check_conflict(action) # add to actions list self._actions.append(action) action.container = self # index the action by any option strings it has for option_string in action.option_strings: self._option_string_actions[option_string] = action # set the flag if any option strings look like negative numbers for option_string in action.option_strings: if self._negative_number_matcher.match(option_string): if not self._has_negative_number_optionals: self._has_negative_number_optionals.append(True) # return the created action return action def _remove_action(self, action): # self._actions.remove(action) list_remove(self._actions, action) def _add_container_actions(self, container): # collect groups by titles title_group_map = {} for group in self._action_groups: if group.title in title_group_map: msg = _('cannot merge actions - two groups are named %r') raise ValueError(msg % (group.title)) title_group_map[group.title] = group # map each action to its group group_map = {} for group in container._action_groups: # if a group with the title exists, use that, otherwise # create a new group matching the container's group if group.title not in title_group_map: title_group_map[group.title] = self.add_argument_group( title=group.title, description=group.description, conflict_handler=group.conflict_handler) # map the actions to their new group for action in group._group_actions: group_map[action] = title_group_map[group.title] # add container's mutually exclusive groups # NOTE: if add_mutually_exclusive_group ever gains title= and # description= then this code will need to be expanded as above for group in container._mutually_exclusive_groups: mutex_group = self.add_mutually_exclusive_group( required=group.required) # map the actions to their new mutex group for action in group._group_actions: group_map[action] = mutex_group # add all actions to this container or their group for action in container._actions: group_map.get(action, self)._add_action(action) def _get_positional_kwargs(self, dest, **kwargs): # make sure required is not specified if 'required' in kwargs: msg = _("'required' is an invalid argument for positionals") raise TypeError(msg) # mark positional arguments as required if at least one is # always required if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: kwargs['required'] = True if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: kwargs['required'] = True # return the keyword arguments with no option strings return dict(kwargs, dest=dest, option_strings=[]) def _get_optional_kwargs(self, *args, **kwargs): # determine short and long option strings option_strings = [] long_option_strings = [] for option_string in args: # error on strings that don't start with an appropriate prefix if not option_string[0] in self.prefix_chars: msg = _('invalid option string %r: ' 'must start with a character %r') tup = option_string, self.prefix_chars raise ValueError(msg % tup) # strings starting with two prefix characters are long options option_strings.append(option_string) if option_string[0] in self.prefix_chars: if len(option_string) > 1: if option_string[1] in self.prefix_chars: long_option_strings.append(option_string) # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' # dest = kwargs.pop('dest', None) dest = dict_pop(kwargs, 'dest', None) if dest is None: if long_option_strings: dest_option_string = long_option_strings[0] else: dest_option_string = option_strings[0] dest = dest_option_string.lstrip(self.prefix_chars) if not dest: msg = _('dest= is required for options like %r') raise ValueError(msg % option_string) dest = dest.replace('-', '_') # return the updated keyword arguments return dict(kwargs, dest=dest, option_strings=option_strings) def _pop_action_class(self, kwargs, default=None): # action = kwargs.pop('action', default) action = dict_pop(kwargs, 'action', default) return self._registry_get('action', action, action) def _get_handler(self): # determine function from conflict handler string handler_func_name = '_handle_conflict_%s' % self.conflict_handler try: return getattr(self, handler_func_name) except AttributeError: msg = _('invalid conflict_resolution value: %r') raise ValueError(msg % self.conflict_handler) def _check_conflict(self, action): # find all options that conflict with this option confl_optionals = [] for option_string in action.option_strings: if option_string in self._option_string_actions: confl_optional = self._option_string_actions[option_string] confl_optionals.append((option_string, confl_optional)) # resolve any conflicts if confl_optionals: conflict_handler = self._get_handler() conflict_handler(action, confl_optionals) def _handle_conflict_error(self, action, conflicting_actions): message = _('conflicting option string(s): %s') conflict_string = ', '.join([option_string for option_string, action in conflicting_actions]) raise ArgumentError(action, message % conflict_string) def _handle_conflict_resolve(self, action, conflicting_actions): # remove all conflicting options for option_string, action in conflicting_actions: # remove the conflicting option # action.option_strings.remove(option_string) list_remove(action.option_strings, option_string) # self._option_string_actions.pop(option_string, None) dict_pop(self._option_string_actions, option_string, None) # if the option now has no option string, remove it from the # container holding it if not action.option_strings: action.container._remove_action(action) class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, **kwargs): # add any missing keyword arguments by checking the container # update = kwargs.setdefault # update('conflict_handler', container.conflict_handler) # update('prefix_chars', container.prefix_chars) # update('argument_default', container.argument_default) setdefault(kwargs, 'conflict_handler', container.conflict_handler) setdefault(kwargs, 'prefix_chars', container.prefix_chars) setdefault(kwargs, 'argument_default', container.argument_default) super_init = super(_ArgumentGroup, self).__init__ super_init(description=description, **kwargs) # group attributes self.title = title self._group_actions = [] # share most attributes with the container self._registries = container._registries self._actions = container._actions self._option_string_actions = container._option_string_actions self._defaults = container._defaults self._has_negative_number_optionals = \ container._has_negative_number_optionals self._mutually_exclusive_groups = container._mutually_exclusive_groups def _add_action(self, action): action = super(_ArgumentGroup, self)._add_action(action) self._group_actions.append(action) return action def _remove_action(self, action): super(_ArgumentGroup, self)._remove_action(action) # self._group_actions.remove(action) self._group_actions = [x for x in self._group_actions if x != action] class _MutuallyExclusiveGroup(_ArgumentGroup): def __init__(self, container, required=False): super(_MutuallyExclusiveGroup, self).__init__(container) self.required = required self._container = container def _add_action(self, action): if action.required: msg = _('mutually exclusive arguments must be optional') raise ValueError(msg) action = self._container._add_action(action) self._group_actions.append(action) return action def _remove_action(self, action): self._container._remove_action(action) # self._group_actions.remove(action) self._group_actions = [x for x in self._group_actions if x != action] class ArgumentParser(_AttributeHolder, _ActionsContainer): """Object for parsing command line strings into Python objects. Keyword Arguments: - prog -- The name of the program (default: sys.argv[0]) - usage -- A usage message (default: auto-generated from arguments) - description -- A description of what the program does - epilog -- Text following the argument descriptions - parents -- Parsers whose arguments should be copied into this one - formatter_class -- HelpFormatter class for printing help messages - prefix_chars -- Characters that prefix optional arguments - fromfile_prefix_chars -- Characters that prefix files containing additional arguments - argument_default -- The default value for all arguments - conflict_handler -- String indicating how to handle conflicts - add_help -- Add a -h/-help option """ def __init__(self, prog=None, usage=None, description=None, epilog=None, version=None, parents=[], formatter_class=HelpFormatter, prefix_chars='-', fromfile_prefix_chars=None, argument_default=None, conflict_handler='error', add_help=True): # if version is not None: # import warnings # warnings.warn( # """The "version" argument to ArgumentParser is deprecated. """ # """Please use """ # """"add_argument(..., action='version', version="N", ...)" """ # """instead""", DeprecationWarning) superinit = super(ArgumentParser, self).__init__ superinit(description=description, prefix_chars=prefix_chars, argument_default=argument_default, conflict_handler=conflict_handler) # default setting for prog if prog is None: prog = _os.path.basename(_sys.argv[0]) self.prog = prog self.usage = usage self.epilog = epilog self.version = version self.formatter_class = formatter_class self.fromfile_prefix_chars = fromfile_prefix_chars self.add_help = add_help add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('optional arguments')) self._subparsers = None # register types def identity(string): return string self.register('type', None, identity) # add help and version arguments if necessary # (using explicit default to override global argument_default) default_prefix = '-' if '-' in prefix_chars else prefix_chars[0] if self.add_help: self.add_argument( default_prefix+'h', default_prefix*2+'help', action='help', default=SUPPRESS, help=_('show this help message and exit')) if self.version: self.add_argument( default_prefix+'v', default_prefix*2+'version', action='version', default=SUPPRESS, version=self.version, help=_("show program's version number and exit")) # add parent arguments and defaults for parent in parents: self._add_container_actions(parent) try: defaults = parent._defaults except AttributeError: pass else: self._defaults.update(defaults) # ======================= # Pretty __repr__ methods # ======================= def _get_kwargs(self): names = [ 'prog', 'usage', 'description', 'version', 'formatter_class', 'conflict_handler', 'add_help', ] return [(name, getattr(self, name)) for name in names] # ================================== # Optional/Positional adding methods # ================================== def add_subparsers(self, **kwargs): if self._subparsers is not None: self.error(_('cannot have multiple subparser arguments')) # add the parser class to the arguments if it's not present # kwargs.setdefault('parser_class', type(self)) setdefault(kwargs, 'parser_class', type(self)) if 'title' in kwargs or 'description' in kwargs: # title = _(kwargs.pop('title', 'subcommands')) title = dict_pop(kwargs, 'title', 'subcommands') # description = _(kwargs.pop('description', None)) description = dict_pop(kwargs, 'description', None) self._subparsers = self.add_argument_group(title, description) else: self._subparsers = self._positionals # prog defaults to the usage message of this parser, skipping # optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: formatter = self._get_formatter() positionals = self._get_positional_actions() groups = self._mutually_exclusive_groups formatter.add_usage(self.usage, positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list parsers_class = self._pop_action_class(kwargs, 'parsers') action = parsers_class(option_strings=[], **kwargs) self._subparsers._add_action(action) # return the created parsers action return action def _add_action(self, action): if action.option_strings: self._optionals._add_action(action) else: self._positionals._add_action(action) return action def _get_optional_actions(self): return [action for action in self._actions if action.option_strings] def _get_positional_actions(self): return [action for action in self._actions if not action.option_strings] # ===================================== # Command line argument parsing methods # ===================================== def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) if argv: msg = _('unrecognized arguments: %s') self.error(msg % ' '.join(argv)) return args def parse_known_args(self, args=None, namespace=None): if args is None: # args default to the system args args = _sys.argv[1:] else: # make sure that args are mutable args = list(args) # default Namespace built from parser defaults if namespace is None: namespace = Namespace() # add any action defaults that aren't present for action in self._actions: if action.dest is not SUPPRESS: if not hasattr(namespace, action.dest): if action.default is not SUPPRESS: setattr(namespace, action.dest, action.default) # add any parser defaults that aren't present for dest in self._defaults: if not hasattr(namespace, dest): setattr(namespace, dest, self._defaults[dest]) # parse the arguments and exit if there are any errors try: namespace, args = self._parse_known_args(args, namespace) if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): # args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) args += (getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) # delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) del namespace.__dict__[_UNRECOGNIZED_ARGS_ATTR] return namespace, args except ArgumentError: err = _sys.exc_info()[1] self.error(str(err)) def _parse_known_args(self, arg_strings, namespace): # replace arg strings that are file references if self.fromfile_prefix_chars is not None: arg_strings = self._read_args_from_files(arg_strings) # map all mutually exclusive arguments to the other arguments # they can't occur with action_conflicts = {} for mutex_group in self._mutually_exclusive_groups: group_actions = mutex_group._group_actions for i, mutex_action in enumerate(mutex_group._group_actions): # conflicts = action_conflicts.setdefault(mutex_action, []) # conflicts.extend(group_actions[:i]) # conflicts.extend(group_actions[i + 1:]) conflicts = setdefault(action_conflicts, mutex_action, []) conflicts += (group_actions[:i]) conflicts += (group_actions[i + 1:]) # find all option indices, and determine the arg_string_pattern # which has an 'O' if there is an option at an index, # an 'A' if there is an argument, or a '-' if there is a '--' option_string_indices = {} arg_string_pattern_parts = [] arg_strings_iter = iter(arg_strings) for i, arg_string in enumerate(arg_strings_iter): # all args after -- are non-options if arg_string == '--': arg_string_pattern_parts.append('-') for arg_string in arg_strings_iter: arg_string_pattern_parts.append('A') # otherwise, add the arg to the arg strings # and note the index if it was an option else: option_tuple = self._parse_optional(arg_string) if option_tuple is None: pattern = 'A' else: option_string_indices[i] = option_tuple pattern = 'O' arg_string_pattern_parts.append(pattern) # join the pieces together to form the pattern arg_strings_pattern = ''.join(arg_string_pattern_parts) # converts arg strings to the appropriate and then takes the action seen_actions = set() seen_non_default_actions = set() def take_action(action, argument_strings, option_string=None): seen_actions.add(action) argument_values = self._get_values(action, argument_strings) # error if this argument is not allowed with other previously # seen arguments, assuming that actions that use the default # value don't really count as "present" if argument_values is not action.default: seen_non_default_actions.add(action) for conflict_action in action_conflicts.get(action, []): if conflict_action in seen_non_default_actions: msg = _('not allowed with argument %s') action_name = _get_action_name(conflict_action) raise ArgumentError(action, msg % action_name) # take the action if we didn't receive a SUPPRESS value # (e.g. from a default) if argument_values is not SUPPRESS: action(self, namespace, argument_values, option_string) # function to convert arg_strings into an optional action def consume_optional(start_index): # get the optional identified at this index option_tuple = option_string_indices[start_index] action, option_string, explicit_arg = option_tuple # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) match_argument = self._match_argument action_tuples = [] while True: # if we found no optional action, skip it if action is None: extras.append(arg_strings[start_index]) return start_index + 1 # if there is an explicit argument, try to match the # optional's string arguments to only this if explicit_arg is not None: arg_count = match_argument(action, 'A') # if the action is a single-dash option and takes no # arguments, try to parse more single-dash options out # of the tail of the option string chars = self.prefix_chars if arg_count == 0 and option_string[1] not in chars: action_tuples.append((action, [], option_string)) char = option_string[0] option_string = char + explicit_arg[0] new_explicit_arg = explicit_arg[1:] or None optionals_map = self._option_string_actions if option_string in optionals_map: action = optionals_map[option_string] explicit_arg = new_explicit_arg else: msg = _('ignored explicit argument %r') raise ArgumentError(action, msg % explicit_arg) # if the action expect exactly one argument, we've # successfully matched the option; exit the loop elif arg_count == 1: stop = start_index + 1 args = [explicit_arg] action_tuples.append((action, args, option_string)) break # error if a double-dash option did not use the # explicit argument else: msg = _('ignored explicit argument %r') raise ArgumentError(action, msg % explicit_arg) # if there is no explicit argument, try to match the # optional's string arguments with the following strings # if successful, exit the loop else: start = start_index + 1 selected_patterns = arg_strings_pattern[start:] arg_count = match_argument(action, selected_patterns) stop = start + arg_count args = arg_strings[start:stop] action_tuples.append((action, args, option_string)) break # add the Optional to the list and return the index at which # the Optional's string args stopped assert action_tuples for action, args, option_string in action_tuples: take_action(action, args, option_string) return stop # the list of Positionals left to be parsed; this is modified # by consume_positionals() positionals = self._get_positional_actions() # function to convert arg_strings into positional actions def consume_positionals(start_index): # match as many Positionals as possible match_partial = self._match_arguments_partial selected_pattern = arg_strings_pattern[start_index:] arg_counts = match_partial(positionals, selected_pattern) # slice off the appropriate arg strings for each Positional # and add the Positional and its args to the list for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] start_index += arg_count take_action(action, args) # slice off the Positionals that we just parsed and return the # index at which the Positionals' string args stopped positionals[:] = positionals[len(arg_counts):] return start_index # consume Positionals and Optionals alternately, until we have # passed the last option string extras = [] start_index = 0 if option_string_indices: max_option_string_index = max(option_string_indices) else: max_option_string_index = -1 while start_index <= max_option_string_index: # consume any Positionals preceding the next option next_option_string_index = min([ index for index in option_string_indices if index >= start_index]) if start_index != next_option_string_index: positionals_end_index = consume_positionals(start_index) # only try to parse the next optional if we didn't consume # the option string during the positionals parsing if positionals_end_index > start_index: start_index = positionals_end_index continue else: start_index = positionals_end_index # if we consumed all the positionals we could and we're not # at the index of an option string, there were extra arguments if start_index not in option_string_indices: strings = arg_strings[start_index:next_option_string_index] # extras.extend(strings) extras += (strings) start_index = next_option_string_index # consume the next optional and any arguments for it start_index = consume_optional(start_index) # consume any positionals following the last Optional stop_index = consume_positionals(start_index) # if we didn't consume all the argument strings, there were extras # extras.extend(arg_strings[stop_index:]) extras += (arg_strings[stop_index:]) # if we didn't use all the Positional objects, there were too few # arg strings supplied. if positionals: self.error(_('too few arguments')) # make sure all required actions were present, and convert defaults. for action in self._actions: if action not in seen_actions: if action.required: name = _get_action_name(action) self.error(_('argument %s is required') % name) else: # Convert action default now instead of doing it before # parsing arguments to avoid calling convert functions # twice (which may fail) if the argument was given, but # only if it was defined already in the namespace if (action.default is not None and isinstance(action.default, basestring) and hasattr(namespace, action.dest) and action.default is getattr(namespace, action.dest)): setattr(namespace, action.dest, self._get_value(action, action.default)) # make sure all required groups had one option present for group in self._mutually_exclusive_groups: if group.required: for action in group._group_actions: if action in seen_non_default_actions: break # if no actions were used, report the error else: names = [_get_action_name(action) for action in group._group_actions if action.help is not SUPPRESS] msg = _('one of the arguments %s is required') self.error(msg % ' '.join(names)) # return the updated namespace and the extra arguments return namespace, extras def _read_args_from_files(self, arg_strings): # expand arguments referencing files new_arg_strings = [] for arg_string in arg_strings: # for regular arguments, just add them back into the list if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: new_arg_strings.append(arg_string) # replace arguments referencing files with the file content else: try: args_file = open(arg_string[1:]) try: arg_strings = [] for arg_line in args_file.read().splitlines(): for arg in self.convert_arg_line_to_args(arg_line): arg_strings.append(arg) arg_strings = self._read_args_from_files(arg_strings) # new_arg_strings.extend(arg_strings) new_arg_strings += (arg_strings) finally: args_file.close() except IOError: err = _sys.exc_info()[1] self.error(str(err)) # return the modified argument list return new_arg_strings def convert_arg_line_to_args(self, arg_line): return [arg_line] def _match_argument(self, action, arg_strings_pattern): # match the pattern for this action to the arg strings nargs_pattern = self._get_nargs_pattern(action) match = _re.match(nargs_pattern, arg_strings_pattern) # raise an exception if we weren't able to find a match if match is None: nargs_errors = { None: _('expected one argument'), OPTIONAL: _('expected at most one argument'), ONE_OR_MORE: _('expected at least one argument'), } default = _('expected %s argument(s)') % action.nargs msg = nargs_errors.get(action.nargs, default) raise ArgumentError(action, msg) # return the number of arguments matched return len(match.group(1)) def _match_arguments_partial(self, actions, arg_strings_pattern): # progressively shorten the actions list by slicing off the # final actions until we find a match result = [] for i in range(len(actions), 0, -1): actions_slice = actions[:i] pattern = ''.join([self._get_nargs_pattern(action) for action in actions_slice]) match = _re.match(pattern, arg_strings_pattern) if match is not None: # result.extend([len(string) for string in match.groups()]) result += ([len(string) for string in match.groups()]) break # return the list of arg string counts return result def _parse_optional(self, arg_string): # if it's an empty string, it was meant to be a positional if not arg_string: return None # if it doesn't start with a prefix, it was meant to be positional if not arg_string[0] in self.prefix_chars: return None # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] return action, arg_string, None # if it's just a single character, it was meant to be positional if len(arg_string) == 1: return None # if the option string before the "=" is present, return the action if '=' in arg_string: option_string, explicit_arg = arg_string.split('=', 1) if option_string in self._option_string_actions: action = self._option_string_actions[option_string] return action, option_string, explicit_arg # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations option_tuples = self._get_option_tuples(arg_string) # if multiple actions match, the option string was ambiguous if len(option_tuples) > 1: options = ', '.join([option_string for action, option_string, explicit_arg in option_tuples]) tup = arg_string, options self.error(_('ambiguous option: %s could match %s') % tup) # if exactly one action matched, this segmentation is good, # so return the parsed action elif len(option_tuples) == 1: option_tuple, = option_tuples return option_tuple # if it was not found as an option, but it looks like a negative # number, it was meant to be positional # unless there are negative-number-like options if self._negative_number_matcher.match(arg_string): if not self._has_negative_number_optionals: return None # if it contains a space, it was meant to be a positional if ' ' in arg_string: return None # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) return None, arg_string, None def _get_option_tuples(self, option_string): result = [] # option strings starting with two prefix characters are only # split at the '=' chars = self.prefix_chars if option_string[0] in chars and option_string[1] in chars: if '=' in option_string: option_prefix, explicit_arg = option_string.split('=', 1) else: option_prefix = option_string explicit_arg = None for option_string in self._option_string_actions: if option_string.startswith(option_prefix): action = self._option_string_actions[option_string] tup = action, option_string, explicit_arg result.append(tup) # single character options can be concatenated with their arguments # but multiple character options always have to have their argument # separate elif option_string[0] in chars and option_string[1] not in chars: option_prefix = option_string explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] for option_string in self._option_string_actions: if option_string == short_option_prefix: action = self._option_string_actions[option_string] tup = action, option_string, short_explicit_arg result.append(tup) elif option_string.startswith(option_prefix): action = self._option_string_actions[option_string] tup = action, option_string, explicit_arg result.append(tup) # shouldn't ever get here else: self.error(_('unexpected option string: %s') % option_string) # return the collected option tuples return result def _get_nargs_pattern(self, action): # in all examples below, we have to allow for '--' args # which are represented as '-' in the pattern nargs = action.nargs # the default (None) is assumed to be a single argument if nargs is None: nargs_pattern = '(-*A-*)' # allow zero or one arguments elif nargs == OPTIONAL: nargs_pattern = '(-*A?-*)' # allow zero or more arguments elif nargs == ZERO_OR_MORE: nargs_pattern = '(-*[A-]*)' # allow one or more arguments elif nargs == ONE_OR_MORE: nargs_pattern = '(-*A[A-]*)' # allow any number of options or arguments elif nargs == REMAINDER: nargs_pattern = '([-AO]*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: nargs_pattern = '(-*A[-AO]*)' # all others should be integers else: nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) # if this is an optional action, -- is not allowed if action.option_strings: nargs_pattern = nargs_pattern.replace('-*', '') nargs_pattern = nargs_pattern.replace('-', '') # return the pattern return nargs_pattern # ======================== # Value conversion methods # ======================== def _get_values(self, action, arg_strings): # for everything but PARSER, REMAINDER args, strip out first '--' if action.nargs not in [PARSER, REMAINDER]: try: # arg_strings.remove('--') arg_strings = [x for x in arg_strings if x != '--'] except ValueError: pass # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: if action.option_strings: value = action.const else: value = action.default if isinstance(value, basestring): value = self._get_value(action, value) self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None elif (not arg_strings and action.nargs == ZERO_OR_MORE and not action.option_strings): if action.default is not None: value = action.default else: value = arg_strings self._check_value(action, value) # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: arg_string, = arg_strings value = self._get_value(action, arg_string) self._check_value(action, value) # REMAINDER arguments convert all values, checking none elif action.nargs == REMAINDER: value = [self._get_value(action, v) for v in arg_strings] # PARSER arguments convert all values, but check only the first elif action.nargs == PARSER: value = [self._get_value(action, v) for v in arg_strings] self._check_value(action, value[0]) # all other types of nargs produce a list else: value = [self._get_value(action, v) for v in arg_strings] for v in value: self._check_value(action, v) # return the converted value return value def _get_value(self, action, arg_string): type_func = self._registry_get('type', action.type, action.type) if not _callable(type_func): msg = _('%r is not callable') raise ArgumentError(action, msg % type_func) # convert the value to the appropriate type try: result = type_func(arg_string) # ArgumentTypeErrors indicate errors except ArgumentTypeError: name = getattr(action.type, '__name__', repr(action.type)) msg = str(_sys.exc_info()[1]) raise ArgumentError(action, msg) # TypeErrors or ValueErrors also indicate errors except (TypeError, ValueError): name = getattr(action.type, '__name__', repr(action.type)) msg = _('invalid %s value: %r') raise ArgumentError(action, msg % (name, arg_string)) # return the converted value return result def _check_value(self, action, value): # converted value must be one of the choices (if specified) if action.choices is not None and value not in action.choices: tup = value, ', '.join(map(repr, action.choices)) msg = _('invalid choice: %r (choose from %s)') % tup raise ArgumentError(action, msg) # ======================= # Help-formatting methods # ======================= def format_usage(self): formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) return formatter.format_help() def format_help(self): formatter = self._get_formatter() # usage formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) # description formatter.add_text(self.description) # positionals, optionals and user-defined groups for action_group in self._action_groups: formatter.start_section(action_group.title) formatter.add_text(action_group.description) formatter.add_arguments(action_group._group_actions) formatter.end_section() # epilog formatter.add_text(self.epilog) # determine help from format above return formatter.format_help() def format_version(self): # import warnings # warnings.warn( # 'The format_version method is deprecated -- the "version" ' # 'argument to ArgumentParser is no longer supported.', # DeprecationWarning) formatter = self._get_formatter() formatter.add_text(self.version) return formatter.format_help() def _get_formatter(self): return self.formatter_class(prog=self.prog) # ===================== # Help-printing methods # ===================== def print_usage(self, file=None): if file is None: file = _sys.stdout self._print_message(self.format_usage(), file) def print_help(self, file=None): if file is None: file = _sys.stdout self._print_message(self.format_help(), file) def print_version(self, file=None): # import warnings # warnings.warn( # 'The print_version method is deprecated -- the "version" ' # 'argument to ArgumentParser is no longer supported.', # DeprecationWarning) self._print_message(self.format_version(), file) def _print_message(self, message, file=None): if message: if file is None: file = _sys.stderr file.write(message) # =============== # Exiting methods # =============== def exit(self, status=0, message=None): if message: self._print_message(message, _sys.stderr) _sys.exit(status) def error(self, message): """error(message: string) Prints a usage message incorporating the message to stderr and exits. If you override this in a subclass, it should not return -- it should either exit or raise an exception. """ self.print_usage(_sys.stderr) self.exit(2, _('%s: error: %s\n') % (self.prog, message)) ================================================ FILE: third_party/stdlib/base64.py ================================================ #! /usr/bin/env python """RFC 3548: Base16, Base32, Base64 Data Encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support import re # import struct import _struct as struct import string import binascii __all__ = [ # Legacy interface exports traditional RFC 1521 Base64 encodings 'encode', 'decode', 'encodestring', 'decodestring', # Generalized interface for other encodings 'b64encode', 'b64decode', 'b32encode', 'b32decode', 'b16encode', 'b16decode', # Standard Base64 encoding 'standard_b64encode', 'standard_b64decode', # Some common Base64 alternatives. As referenced by RFC 3458, see thread # starting at: # # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html 'urlsafe_b64encode', 'urlsafe_b64decode', ] _translation = [chr(_x) for _x in range(256)] EMPTYSTRING = '' def _translate(s, altchars): translation = _translation[:] for k, v in altchars.items(): translation[ord(k)] = v return s.translate(''.join(translation)) # Base64 encoding/decoding uses binascii def b64encode(s, altchars=None): """Encode a string using Base64. s is the string to encode. Optional altchars must be a string of at least length 2 (additional characters are ignored) which specifies an alternative alphabet for the '+' and '/' characters. This allows an application to e.g. generate url or filesystem safe Base64 strings. The encoded string is returned. """ # Strip off the trailing newline encoded = binascii.b2a_base64(s)[:-1] if altchars is not None: return encoded.translate(string.maketrans(b'+/', altchars[:2])) return encoded def b64decode(s, altchars=None): """Decode a Base64 encoded string. s is the string to decode. Optional altchars must be a string of at least length 2 (additional characters are ignored) which specifies the alternative alphabet used instead of the '+' and '/' characters. The decoded string is returned. A TypeError is raised if s is incorrectly padded. Characters that are neither in the normal base-64 alphabet nor the alternative alphabet are discarded prior to the padding check. """ if altchars is not None: s = s.translate(string.maketrans(altchars[:2], '+/')) try: return binascii.a2b_base64(s) except binascii.Error, msg: # Transform this exception for consistency raise TypeError(msg) def standard_b64encode(s): """Encode a string using the standard Base64 alphabet. s is the string to encode. The encoded string is returned. """ return b64encode(s) def standard_b64decode(s): """Decode a string encoded with the standard Base64 alphabet. Argument s is the string to decode. The decoded string is returned. A TypeError is raised if the string is incorrectly padded. Characters that are not in the standard alphabet are discarded prior to the padding check. """ return b64decode(s) _urlsafe_encode_translation = string.maketrans(b'+/', b'-_') _urlsafe_decode_translation = string.maketrans(b'-_', b'+/') def urlsafe_b64encode(s): """Encode a string using the URL- and filesystem-safe Base64 alphabet. Argument s is the string to encode. The encoded string is returned. The alphabet uses '-' instead of '+' and '_' instead of '/'. """ return b64encode(s).translate(_urlsafe_encode_translation) def urlsafe_b64decode(s): """Decode a string using the URL- and filesystem-safe Base64 alphabet. Argument s is the string to decode. The decoded string is returned. A TypeError is raised if the string is incorrectly padded. Characters that are not in the URL-safe base-64 alphabet, and are not a plus '+' or slash '/', are discarded prior to the padding check. The alphabet uses '-' instead of '+' and '_' instead of '/'. """ return b64decode(s.translate(_urlsafe_decode_translation)) # Base32 encoding/decoding must be done in Python _b32alphabet = { 0: 'A', 9: 'J', 18: 'S', 27: '3', 1: 'B', 10: 'K', 19: 'T', 28: '4', 2: 'C', 11: 'L', 20: 'U', 29: '5', 3: 'D', 12: 'M', 21: 'V', 30: '6', 4: 'E', 13: 'N', 22: 'W', 31: '7', 5: 'F', 14: 'O', 23: 'X', 6: 'G', 15: 'P', 24: 'Y', 7: 'H', 16: 'Q', 25: 'Z', 8: 'I', 17: 'R', 26: '2', } _b32tab = _b32alphabet.items() _b32tab.sort() _b32tab = [v for k, v in _b32tab] _b32rev = dict([(v, long(k)) for k, v in _b32alphabet.items()]) def b32encode(s): """Encode a string using Base32. s is the string to encode. The encoded string is returned. """ parts = [] quanta, leftover = divmod(len(s), 5) # Pad the last quantum with zero bits if necessary if leftover: s += ('\0' * (5 - leftover)) quanta += 1 for i in range(quanta): # c1 and c2 are 16 bits wide, c3 is 8 bits wide. The intent of this # code is to process the 40 bits in units of 5 bits. So we take the 1 # leftover bit of c1 and tack it onto c2. Then we take the 2 leftover # bits of c2 and tack them onto c3. The shifts and masks are intended # to give us values of exactly 5 bits in width. c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5]) c2 += (c1 & 1) << 16 # 17 bits wide c3 += (c2 & 3) << 8 # 10 bits wide parts.extend([_b32tab[c1 >> 11], # bits 1 - 5 _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10 _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15 _b32tab[c2 >> 12], # bits 16 - 20 (1 - 5) _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10) _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15) _b32tab[c3 >> 5], # bits 31 - 35 (1 - 5) _b32tab[c3 & 0x1f], # bits 36 - 40 (1 - 5) ]) encoded = EMPTYSTRING.join(parts) # Adjust for any leftover partial quanta if leftover == 1: return encoded[:-6] + '======' elif leftover == 2: return encoded[:-4] + '====' elif leftover == 3: return encoded[:-3] + '===' elif leftover == 4: return encoded[:-1] + '=' return encoded def b32decode(s, casefold=False, map01=None): """Decode a Base32 encoded string. s is the string to decode. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. RFC 3548 allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument map01 when not None, specifies which letter the digit 1 should be mapped to (when map01 is not None, the digit 0 is always mapped to the letter O). For security purposes the default is None, so that 0 and 1 are not allowed in the input. The decoded string is returned. A TypeError is raised if s were incorrectly padded or if there are non-alphabet characters present in the string. """ quanta, leftover = divmod(len(s), 8) if leftover: raise TypeError('Incorrect padding') # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be # either L (el) or I (eye). if map01: s = s.translate(string.maketrans(b'01', b'O' + map01)) if casefold: s = s.upper() # Strip off pad characters from the right. We need to count the pad # characters because this will tell us how many null bytes to remove from # the end of the decoded string. padchars = 0 mo = re.search('(?P[=]*)$', s) if mo: padchars = len(mo.group('pad')) if padchars > 0: s = s[:-padchars] # Now decode the full quanta parts = [] acc = 0 shift = 35 for c in s: val = _b32rev.get(c) if val is None: raise TypeError('Non-base32 digit found') acc += _b32rev[c] << shift shift -= 5 if shift < 0: parts.append(binascii.unhexlify('%010x' % acc)) acc = 0 shift = 35 # Process the last, partial quanta last = binascii.unhexlify('%010x' % acc) if padchars == 0: last = '' # No characters elif padchars == 1: last = last[:-1] elif padchars == 3: last = last[:-2] elif padchars == 4: last = last[:-3] elif padchars == 6: last = last[:-4] else: raise TypeError('Incorrect padding') parts.append(last) return EMPTYSTRING.join(parts) # RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. def b16encode(s): """Encode a string using Base16. s is the string to encode. The encoded string is returned. """ return binascii.hexlify(s).upper() def b16decode(s, casefold=False): """Decode a Base16 encoded string. s is the string to decode. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. The decoded string is returned. A TypeError is raised if s is incorrectly padded or if there are non-alphabet characters present in the string. """ if casefold: s = s.upper() if re.search('[^0-9A-F]', s): raise TypeError('Non-base16 digit found') return binascii.unhexlify(s) # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it # though. MAXLINESIZE = 76 # Excluding the CRLF MAXBINSIZE = (MAXLINESIZE//4)*3 def encode(input, output): """Encode a file.""" while True: s = input.read(MAXBINSIZE) if not s: break while len(s) < MAXBINSIZE: ns = input.read(MAXBINSIZE-len(s)) if not ns: break s += ns line = binascii.b2a_base64(s) output.write(line) def decode(input, output): """Decode a file.""" while True: line = input.readline() if not line: break s = binascii.a2b_base64(line) output.write(s) def encodestring(s): """Encode a string into multiple lines of base-64 data.""" pieces = [] for i in range(0, len(s), MAXBINSIZE): chunk = s[i : i + MAXBINSIZE] pieces.append(binascii.b2a_base64(chunk)) return "".join(pieces) def decodestring(s): """Decode a string.""" return binascii.a2b_base64(s) # Useable as a script... def test(): """Small test program""" import sys, getopt try: opts, args = getopt.getopt(sys.argv[1:], 'deut') except getopt.error, msg: sys.stdout = sys.stderr print msg print """usage: %s [-d|-e|-u|-t] [file|-] -d, -u: decode -e: encode (default) -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0] sys.exit(2) func = encode for o, a in opts: if o == '-e': func = encode if o == '-d': func = decode if o == '-u': func = decode if o == '-t': test1(); return if args and args[0] != '-': with open(args[0], 'rb') as f: func(f, sys.stdout) else: func(sys.stdin, sys.stdout) def test1(): s0 = "Aladdin:open sesame" s1 = encodestring(s0) s2 = decodestring(s1) print s0, repr(s1), s2 if __name__ == '__main__': test() ================================================ FILE: third_party/stdlib/bisect.py ================================================ """Bisection algorithms.""" def insort_right(a, x, lo=0, hi=None): """Insert item x in list a, and keep it sorted assuming a is sorted. If x is already in a, insert it to the right of the rightmost x. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. """ if lo < 0: raise ValueError('lo must be non-negative') if hi is None: hi = len(a) while lo < hi: mid = (lo+hi)//2 if x < a[mid]: hi = mid else: lo = mid+1 a.insert(lo, x) insort = insort_right # backward compatibility def bisect_right(a, x, lo=0, hi=None): """Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e <= x, and all e in a[i:] have e > x. So if x already appears in the list, a.insert(x) will insert just after the rightmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. """ if lo < 0: raise ValueError('lo must be non-negative') if hi is None: hi = len(a) while lo < hi: mid = (lo+hi)//2 if x < a[mid]: hi = mid else: lo = mid+1 return lo bisect = bisect_right # backward compatibility def insort_left(a, x, lo=0, hi=None): """Insert item x in list a, and keep it sorted assuming a is sorted. If x is already in a, insert it to the left of the leftmost x. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. """ if lo < 0: raise ValueError('lo must be non-negative') if hi is None: hi = len(a) while lo < hi: mid = (lo+hi)//2 if a[mid] < x: lo = mid+1 else: hi = mid a.insert(lo, x) def bisect_left(a, x, lo=0, hi=None): """Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e < x, and all e in a[i:] have e >= x. So if x already appears in the list, a.insert(x) will insert just before the leftmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. """ if lo < 0: raise ValueError('lo must be non-negative') if hi is None: hi = len(a) while lo < hi: mid = (lo+hi)//2 if a[mid] < x: lo = mid+1 else: hi = mid return lo # Overwrite above definitions with a fast C implementation # try: # from _bisect import * # except ImportError: # pass ================================================ FILE: third_party/stdlib/collections.py ================================================ '''This module implements specialized container datatypes providing alternatives to Python's general purpose built-in containers, dict, list, set, and tuple. * namedtuple factory function for creating tuple subclasses with named fields * deque list-like container with fast appends and pops on either end * Counter dict subclass for counting hashable objects * OrderedDict dict subclass that remembers the order entries were added * defaultdict dict subclass that calls a factory function to supply missing values ''' __all__ = ['Counter', 'namedtuple', 'OrderedDict', 'deque'] # 'deque', 'defaultdict', # For bootstrapping reasons, the collection ABCs are defined in _abcoll.py. # They should however be considered an integral part of collections.py. import _abcoll # TODO: Support from foo import * syntax. for name in _abcoll.__all__: globals()[name] = getattr(_abcoll, name) import _collections deque = _collections.deque #defaultdict = _collections.defaultdict import operator _itemgetter = operator.itemgetter _eq = operator.eq import keyword _iskeyword = keyword.iskeyword import sys as _sys import heapq as _heapq import itertools _repeat = itertools.repeat _chain = itertools.chain _starmap = itertools.starmap _imap = itertools.imap #try: import thread _get_ident = thread.get_ident #except ImportError: # from dummy_thread import get_ident as _get_ident ################################################################################ ### OrderedDict ################################################################################ class OrderedDict(dict): 'Dictionary that remembers insertion order' # An inherited dict maps keys to values. # The inherited dict provides __getitem__, __len__, __contains__, and get. # The remaining methods are order-aware. # Big-O running times for all methods are the same as regular dictionaries. # The internal self.__map dict maps keys to links in a doubly linked list. # The circular doubly linked list starts and ends with a sentinel element. # The sentinel element never gets deleted (this simplifies the algorithm). # Each link is stored as a list of length three: [PREV, NEXT, KEY]. def __init__(*args, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries, but keyword arguments are not recommended because their insertion order is arbitrary. ''' if not args: raise TypeError("descriptor '__init__' of 'OrderedDict' object " "needs an argument") self = args[0] args = args[1:] if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) try: self.__root except AttributeError: self.__root = root = [] # sentinel node root[:] = [root, root, None] self.__map = {} self.__update(*args, **kwds) def __setitem__(self, key, value, dict_setitem=dict.__setitem__): 'od.__setitem__(i, y) <==> od[i]=y' # Setting a new item creates a new link at the end of the linked list, # and the inherited dictionary is updated with the new key/value pair. if key not in self: root = self.__root last = root[0] last[1] = root[0] = self.__map[key] = [last, root, key] return dict_setitem(self, key, value) def __delitem__(self, key, dict_delitem=dict.__delitem__): 'od.__delitem__(y) <==> del od[y]' # Deleting an existing item uses self.__map to find the link which gets # removed by updating the links in the predecessor and successor nodes. dict_delitem(self, key) link_prev, link_next, _ = self.__map.pop(key) link_prev[1] = link_next # update link_prev[NEXT] link_next[0] = link_prev # update link_next[PREV] def __iter__(self): 'od.__iter__() <==> iter(od)' # Traverse the linked list in order. root = self.__root curr = root[1] # start at the first node while curr is not root: yield curr[2] # yield the curr[KEY] curr = curr[1] # move to next node def __reversed__(self): 'od.__reversed__() <==> reversed(od)' # Traverse the linked list in reverse order. root = self.__root curr = root[0] # start at the last node while curr is not root: yield curr[2] # yield the curr[KEY] curr = curr[0] # move to previous node def clear(self): 'od.clear() -> None. Remove all items from od.' root = self.__root root[:] = [root, root, None] self.__map.clear() dict.clear(self) # -- the following methods do not depend on the internal structure -- def keys(self): 'od.keys() -> list of keys in od' return list(self) def values(self): 'od.values() -> list of values in od' return [self[key] for key in self] def items(self): 'od.items() -> list of (key, value) pairs in od' return [(key, self[key]) for key in self] def iterkeys(self): 'od.iterkeys() -> an iterator over the keys in od' return iter(self) def itervalues(self): 'od.itervalues -> an iterator over the values in od' for k in self: yield self[k] def iteritems(self): 'od.iteritems -> an iterator over the (key, value) pairs in od' for k in self: yield (k, self[k]) update = MutableMapping.update __update = update # let subclasses override update without breaking __init__ __marker = object() def pop(self, key, default=__marker): '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised. ''' if key in self: result = self[key] del self[key] return result if default is self.__marker: raise KeyError(key) return default def setdefault(self, key, default=None): 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' if key in self: return self[key] self[key] = default return default def popitem(self, last=True): '''od.popitem() -> (k, v), return and remove a (key, value) pair. Pairs are returned in LIFO order if last is true or FIFO order if false. ''' if not self: raise KeyError('dictionary is empty') key = next(reversed(self) if last else iter(self)) value = self.pop(key) return key, value def __repr__(self, _repr_running={}): 'od.__repr__() <==> repr(od)' call_key = id(self), _get_ident() if call_key in _repr_running: return '...' _repr_running[call_key] = 1 try: if not self: return '%s()' % (self.__class__.__name__,) return '%s(%r)' % (self.__class__.__name__, self.items()) finally: del _repr_running[call_key] def __reduce__(self): 'Return state information for pickling' items = [[k, self[k]] for k in self] inst_dict = vars(self).copy() for k in vars(OrderedDict()): inst_dict.pop(k, None) if inst_dict: return (self.__class__, (items,), inst_dict) return self.__class__, (items,) def copy(self): 'od.copy() -> a shallow copy of od' return self.__class__(self) @classmethod def fromkeys(cls, iterable, value=None): '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S. If not specified, the value defaults to None. ''' self = cls() for key in iterable: self[key] = value return self def __eq__(self, other): '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive while comparison to a regular mapping is order-insensitive. ''' if isinstance(other, OrderedDict): return dict.__eq__(self, other) and all(_imap(_eq, self, other)) return dict.__eq__(self, other) def __ne__(self, other): 'od.__ne__(y) <==> od!=y' return not self == other # -- the following methods support python 3.x style dictionary views -- def viewkeys(self): "od.viewkeys() -> a set-like object providing a view on od's keys" return KeysView(self) def viewvalues(self): "od.viewvalues() -> an object providing a view on od's values" return ValuesView(self) def viewitems(self): "od.viewitems() -> a set-like object providing a view on od's items" return ItemsView(self) ################################################################################ ### namedtuple ################################################################################ _class_template = '''\ class {typename}(tuple): '{typename}({arg_list})' __slots__ = () _fields = {field_names!r} def __new__(_cls, {arg_list}): 'Create new instance of {typename}({arg_list})' return _tuple.__new__(_cls, ({arg_list})) @classmethod def _make(cls, iterable, new=tuple.__new__, len=len): 'Make a new {typename} object from a sequence or iterable' result = new(cls, iterable) if len(result) != {num_fields:d}: raise TypeError('Expected {num_fields:d} arguments, got %d' % len(result)) return result def __repr__(self): 'Return a nicely formatted representation string' return '{typename}({repr_fmt})' % self def _asdict(self): 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) def _replace(_self, **kwds): 'Return a new {typename} object replacing specified fields with new values' result = _self._make(map(kwds.pop, {field_names!r}, _self)) if kwds: raise ValueError('Got unexpected field names: %r' % kwds.keys()) return result def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) __dict__ = _property(_asdict) def __getstate__(self): 'Exclude the OrderedDict from pickling' pass {field_defs} ''' _repr_template = '{name}=%r' _field_template = '''\ {name} = _property(_itemgetter({index:d}), doc='Alias for field number {index:d}') ''' #def namedtuple(typename, field_names, verbose=False, rename=False): # """Returns a new subclass of tuple with named fields. # # >>> Point = namedtuple('Point', ['x', 'y']) # >>> Point.__doc__ # docstring for the new class # 'Point(x, y)' # >>> p = Point(11, y=22) # instantiate with positional args or keywords # >>> p[0] + p[1] # indexable like a plain tuple # 33 # >>> x, y = p # unpack like a regular tuple # >>> x, y # (11, 22) # >>> p.x + p.y # fields also accessible by name # 33 # >>> d = p._asdict() # convert to a dictionary # >>> d['x'] # 11 # >>> Point(**d) # convert from a dictionary # Point(x=11, y=22) # >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields # Point(x=100, y=22) # # """ # # # Validate the field names. At the user's option, either generate an error # # message or automatically replace the field name with a valid name. # if isinstance(field_names, basestring): # field_names = field_names.replace(',', ' ').split() # field_names = map(str, field_names) # typename = str(typename) # if rename: # seen = set() # for index, name in enumerate(field_names): # if (not all(c.isalnum() or c=='_' for c in name) # or _iskeyword(name) # or not name # or name[0].isdigit() # or name.startswith('_') # or name in seen): # field_names[index] = '_%d' % index # seen.add(name) # for name in [typename] + field_names: # if type(name) != str: # raise TypeError('Type names and field names must be strings') # if not all(c.isalnum() or c=='_' for c in name): # raise ValueError('Type names and field names can only contain ' # 'alphanumeric characters and underscores: %r' % name) # if _iskeyword(name): # raise ValueError('Type names and field names cannot be a ' # 'keyword: %r' % name) # if name[0].isdigit(): # raise ValueError('Type names and field names cannot start with ' # 'a number: %r' % name) # seen = set() # for name in field_names: # if name.startswith('_') and not rename: # raise ValueError('Field names cannot start with an underscore: ' # '%r' % name) # if name in seen: # raise ValueError('Encountered duplicate field name: %r' % name) # seen.add(name) # # # Fill-in the class template # class_definition = _class_template.format( # typename = typename, # field_names = tuple(field_names), # num_fields = len(field_names), # arg_list = repr(tuple(field_names)).replace("'", "")[1:-1], # repr_fmt = ', '.join(_repr_template.format(name=name) # for name in field_names), # field_defs = '\n'.join(_field_template.format(index=index, name=name) # for index, name in enumerate(field_names)) # ) # if verbose: # print class_definition # # # Execute the template string in a temporary namespace and support # # tracing utilities by setting a value for frame.f_globals['__name__'] # namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, # OrderedDict=OrderedDict, _property=property, _tuple=tuple) # try: # exec class_definition in namespace # except SyntaxError as e: # raise SyntaxError(e.message + ':\n' + class_definition) # result = namespace[typename] # # # For pickling to work, the __module__ variable needs to be set to the frame # # where the named tuple is created. Bypass this step in environments where # # sys._getframe is not defined (Jython for example) or sys._getframe is not # # defined for arguments greater than 0 (IronPython). # try: # result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') # except (AttributeError, ValueError): # pass # # return result ######################################################################## ### Counter ######################################################################## class Counter(dict): '''Dict subclass for counting hashable items. Sometimes called a bag or multiset. Elements are stored as dictionary keys and their counts are stored as dictionary values. >>> c = Counter('abcdeabcdabcaba') # count elements from a string >>> c.most_common(3) # three most common elements [('a', 5), ('b', 4), ('c', 3)] >>> sorted(c) # list all unique elements ['a', 'b', 'c', 'd', 'e'] >>> ''.join(sorted(c.elements())) # list elements with repetitions 'aaaaabbbbcccdde' >>> sum(c.values()) # total of all counts 15 >>> c['a'] # count of letter 'a' 5 >>> for elem in 'shazam': # update counts from an iterable ... c[elem] += 1 # by adding 1 to each element's count >>> c['a'] # now there are seven 'a' 7 >>> del c['b'] # remove all 'b' >>> c['b'] # now there are zero 'b' 0 >>> d = Counter('simsalabim') # make another counter >>> c.update(d) # add in the second counter >>> c['a'] # now there are nine 'a' 9 >>> c.clear() # empty the counter >>> c Counter() Note: If a count is set to zero or reduced to zero, it will remain in the counter until the entry is deleted or the counter is cleared: >>> c = Counter('aaabbc') >>> c['b'] -= 2 # reduce the count of 'b' by two >>> c.most_common() # 'b' is still in, but its count is zero [('a', 3), ('c', 1), ('b', 0)] ''' # References: # http://en.wikipedia.org/wiki/Multiset # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm # http://code.activestate.com/recipes/259174/ # Knuth, TAOCP Vol. II section 4.6.3 def __init__(*args, **kwds): '''Create a new, empty Counter object. And if given, count elements from an input iterable. Or, initialize the count from another mapping of elements to their counts. >>> c = Counter() # a new, empty counter >>> c = Counter('gallahad') # a new counter from an iterable >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping >>> c = Counter(a=4, b=2) # a new counter from keyword args ''' if not args: raise TypeError("descriptor '__init__' of 'Counter' object " "needs an argument") self = args[0] args = args[1:] if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) super(Counter, self).__init__() self.update(*args, **kwds) def __missing__(self, key): 'The count of elements not in the Counter is zero.' # Needed so that self[missing_item] does not raise KeyError return 0 def most_common(self, n=None): '''List the n most common elements and their counts from the most common to the least. If n is None, then list all element counts. >>> Counter('abcdeabcdabcaba').most_common(3) [('a', 5), ('b', 4), ('c', 3)] ''' # Emulate Bag.sortedByCount from Smalltalk if n is None: return sorted(self.iteritems(), key=_itemgetter(1), reverse=True) return _heapq.nlargest(n, self.iteritems(), key=_itemgetter(1)) def elements(self): '''Iterator over elements repeating each as many times as its count. >>> c = Counter('ABCABC') >>> sorted(c.elements()) ['A', 'A', 'B', 'B', 'C', 'C'] # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) >>> product = 1 >>> for factor in prime_factors.elements(): # loop over factors ... product *= factor # and multiply them >>> product 1836 Note, if an element's count has been set to zero or is a negative number, elements() will ignore it. ''' # Emulate Bag.do from Smalltalk and Multiset.begin from C++. return _chain.from_iterable(_starmap(_repeat, self.iteritems())) # Override dict methods where necessary @classmethod def fromkeys(cls, iterable, v=None): # There is no equivalent method for counters because setting v=1 # means that no element can have a count greater than one. raise NotImplementedError( 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') def update(*args, **kwds): '''Like dict.update() but add counts instead of replacing them. Source can be an iterable, a dictionary, or another Counter instance. >>> c = Counter('which') >>> c.update('witch') # add elements from another iterable >>> d = Counter('watch') >>> c.update(d) # add elements from another counter >>> c['h'] # four 'h' in which, witch, and watch 4 ''' # The regular dict.update() operation makes no sense here because the # replace behavior results in the some of original untouched counts # being mixed-in with all of the other counts for a mismash that # doesn't have a straight-forward interpretation in most counting # contexts. Instead, we implement straight-addition. Both the inputs # and outputs are allowed to contain zero and negative counts. if not args: raise TypeError("descriptor 'update' of 'Counter' object " "needs an argument") self = args[0] args = args[1:] if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) iterable = args[0] if args else None if iterable is not None: if isinstance(iterable, Mapping): if self: self_get = self.get for elem, count in iterable.iteritems(): self[elem] = self_get(elem, 0) + count else: super(Counter, self).update(iterable) # fast path when counter is empty else: self_get = self.get for elem in iterable: self[elem] = self_get(elem, 0) + 1 if kwds: self.update(kwds) def subtract(*args, **kwds): '''Like dict.update() but subtracts counts instead of replacing them. Counts can be reduced below zero. Both the inputs and outputs are allowed to contain zero and negative counts. Source can be an iterable, a dictionary, or another Counter instance. >>> c = Counter('which') >>> c.subtract('witch') # subtract elements from another iterable >>> c.subtract(Counter('watch')) # subtract elements from another counter >>> c['h'] # 2 in which, minus 1 in witch, minus 1 in watch 0 >>> c['w'] # 1 in which, minus 1 in witch, minus 1 in watch -1 ''' if not args: raise TypeError("descriptor 'subtract' of 'Counter' object " "needs an argument") self = args[0] args = args[1:] if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) iterable = args[0] if args else None if iterable is not None: self_get = self.get if isinstance(iterable, Mapping): for elem, count in iterable.items(): self[elem] = self_get(elem, 0) - count else: for elem in iterable: self[elem] = self_get(elem, 0) - 1 if kwds: self.subtract(kwds) def copy(self): 'Return a shallow copy.' return self.__class__(self) def __reduce__(self): return self.__class__, (dict(self),) def __delitem__(self, elem): 'Like dict.__delitem__() but does not raise KeyError for missing values.' if elem in self: super(Counter, self).__delitem__(elem) def __repr__(self): if not self: return '%s()' % self.__class__.__name__ items = ', '.join(map('%r: %r'.__mod__, self.most_common())) return '%s({%s})' % (self.__class__.__name__, items) # Multiset-style mathematical operations discussed in: # Knuth TAOCP Volume II section 4.6.3 exercise 19 # and at http://en.wikipedia.org/wiki/Multiset # # Outputs guaranteed to only include positive counts. # # To strip negative and zero counts, add-in an empty counter: # c += Counter() def __add__(self, other): '''Add counts from two counters. >>> Counter('abbb') + Counter('bcc') Counter({'b': 4, 'c': 2, 'a': 1}) ''' if not isinstance(other, Counter): return NotImplemented result = Counter() for elem, count in self.items(): newcount = count + other[elem] if newcount > 0: result[elem] = newcount for elem, count in other.items(): if elem not in self and count > 0: result[elem] = count return result def __sub__(self, other): ''' Subtract count, but keep only results with positive counts. >>> Counter('abbbc') - Counter('bccd') Counter({'b': 2, 'a': 1}) ''' if not isinstance(other, Counter): return NotImplemented result = Counter() for elem, count in self.items(): newcount = count - other[elem] if newcount > 0: result[elem] = newcount for elem, count in other.items(): if elem not in self and count < 0: result[elem] = 0 - count return result def __or__(self, other): '''Union is the maximum of value in either of the input counters. >>> Counter('abbb') | Counter('bcc') Counter({'b': 3, 'c': 2, 'a': 1}) ''' if not isinstance(other, Counter): return NotImplemented result = Counter() for elem, count in self.items(): other_count = other[elem] newcount = other_count if count < other_count else count if newcount > 0: result[elem] = newcount for elem, count in other.items(): if elem not in self and count > 0: result[elem] = count return result def __and__(self, other): ''' Intersection is the minimum of corresponding counts. >>> Counter('abbb') & Counter('bcc') Counter({'b': 1}) ''' if not isinstance(other, Counter): return NotImplemented result = Counter() for elem, count in self.items(): other_count = other[elem] newcount = count if count < other_count else other_count if newcount > 0: result[elem] = newcount return result if __name__ == '__main__': pass # # verify that instances can be pickled # from cPickle import loads, dumps # Point = namedtuple('Point', 'x, y', True) # p = Point(x=10, y=20) # assert p == loads(dumps(p)) # # # test and demonstrate ability to override methods # class Point(namedtuple('Point', 'x y')): # __slots__ = () # @property # def hypot(self): # return (self.x ** 2 + self.y ** 2) ** 0.5 # def __str__(self): # return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) # # for p in Point(3, 4), Point(14, 5/7.): # print p # # class Point(namedtuple('Point', 'x y')): # 'Point class with optimized _make() and _replace() without error-checking' # __slots__ = () # _make = classmethod(tuple.__new__) # def _replace(self, _map=map, **kwds): # return self._make(_map(kwds.get, ('x', 'y'), self)) # # print Point(11, 22)._replace(x=100) # # Point3D = namedtuple('Point3D', Point._fields + ('z',)) # print Point3D.__doc__ # # import doctest # TestResults = namedtuple('TestResults', 'failed attempted') # print TestResults(*doctest.testmod()) ================================================ FILE: third_party/stdlib/colorsys.py ================================================ """Conversion functions between RGB and other color systems. This modules provides two functions for each color system ABC: rgb_to_abc(r, g, b) --> a, b, c abc_to_rgb(a, b, c) --> r, g, b All inputs and outputs are triples of floats in the range [0.0...1.0] (with the exception of I and Q, which covers a slightly larger range). Inputs outside the valid range may cause exceptions or invalid outputs. Supported color systems: RGB: Red, Green, Blue components YIQ: Luminance, Chrominance (used by composite video signals) HLS: Hue, Luminance, Saturation HSV: Hue, Saturation, Value """ # References: # http://en.wikipedia.org/wiki/YIQ # http://en.wikipedia.org/wiki/HLS_color_space # http://en.wikipedia.org/wiki/HSV_color_space __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb", "rgb_to_hsv","hsv_to_rgb"] # Some floating point constants ONE_THIRD = 1.0/3.0 ONE_SIXTH = 1.0/6.0 TWO_THIRD = 2.0/3.0 # YIQ: used by composite video signals (linear combinations of RGB) # Y: perceived grey level (0.0 == black, 1.0 == white) # I, Q: color components # # There are a great many versions of the constants used in these formulae. # The ones in this library uses constants from the FCC version of NTSC. def rgb_to_yiq(r, g, b): y = 0.30*r + 0.59*g + 0.11*b i = 0.74*(r-y) - 0.27*(b-y) q = 0.48*(r-y) + 0.41*(b-y) return (y, i, q) def yiq_to_rgb(y, i, q): # r = y + (0.27*q + 0.41*i) / (0.74*0.41 + 0.27*0.48) # b = y + (0.74*q - 0.48*i) / (0.74*0.41 + 0.27*0.48) # g = y - (0.30*(r-y) + 0.11*(b-y)) / 0.59 r = y + 0.9468822170900693*i + 0.6235565819861433*q g = y - 0.27478764629897834*i - 0.6356910791873801*q b = y - 1.1085450346420322*i + 1.7090069284064666*q if r < 0.0: r = 0.0 if g < 0.0: g = 0.0 if b < 0.0: b = 0.0 if r > 1.0: r = 1.0 if g > 1.0: g = 1.0 if b > 1.0: b = 1.0 return (r, g, b) # HLS: Hue, Luminance, Saturation # H: position in the spectrum # L: color lightness # S: color saturation def rgb_to_hls(r, g, b): maxc = max(r, g, b) minc = min(r, g, b) # XXX Can optimize (maxc+minc) and (maxc-minc) l = (minc+maxc)/2.0 if minc == maxc: return 0.0, l, 0.0 if l <= 0.5: s = (maxc-minc) / (maxc+minc) else: s = (maxc-minc) / (2.0-maxc-minc) rc = (maxc-r) / (maxc-minc) gc = (maxc-g) / (maxc-minc) bc = (maxc-b) / (maxc-minc) if r == maxc: h = bc-gc elif g == maxc: h = 2.0+rc-bc else: h = 4.0+gc-rc h = (h/6.0) % 1.0 return h, l, s def hls_to_rgb(h, l, s): if s == 0.0: return l, l, l if l <= 0.5: m2 = l * (1.0+s) else: m2 = l+s-(l*s) m1 = 2.0*l - m2 return (_v(m1, m2, h+ONE_THIRD), _v(m1, m2, h), _v(m1, m2, h-ONE_THIRD)) def _v(m1, m2, hue): hue = hue % 1.0 if hue < ONE_SIXTH: return m1 + (m2-m1)*hue*6.0 if hue < 0.5: return m2 if hue < TWO_THIRD: return m1 + (m2-m1)*(TWO_THIRD-hue)*6.0 return m1 # HSV: Hue, Saturation, Value # H: position in the spectrum # S: color saturation ("purity") # V: color brightness def rgb_to_hsv(r, g, b): maxc = max(r, g, b) minc = min(r, g, b) v = maxc if minc == maxc: return 0.0, 0.0, v s = (maxc-minc) / maxc rc = (maxc-r) / (maxc-minc) gc = (maxc-g) / (maxc-minc) bc = (maxc-b) / (maxc-minc) if r == maxc: h = bc-gc elif g == maxc: h = 2.0+rc-bc else: h = 4.0+gc-rc h = (h/6.0) % 1.0 return h, s, v def hsv_to_rgb(h, s, v): if s == 0.0: return v, v, v i = int(h*6.0) # XXX assume int() truncates! f = (h*6.0) - i p = v*(1.0 - s) q = v*(1.0 - s*f) t = v*(1.0 - s*(1.0-f)) i = i%6 if i == 0: return v, t, p if i == 1: return q, v, p if i == 2: return p, v, t if i == 3: return p, q, v if i == 4: return t, p, v if i == 5: return v, p, q ================================================ FILE: third_party/stdlib/contextlib.py ================================================ """Utilities for with-statement contexts. See PEP 343.""" import sys # from functools import wraps import functools wraps = functools.wraps # from warnings import warn import warnings warn = warnings.warn __all__ = ["contextmanager", "nested", "closing"] class GeneratorContextManager(object): """Helper for @contextmanager decorator.""" def __init__(self, gen): self.gen = gen def __enter__(self): try: return self.gen.next() except StopIteration: raise RuntimeError("generator didn't yield") def __exit__(self, t, value, tb): if t is None: try: self.gen.next() except StopIteration: return else: raise RuntimeError("generator didn't stop") else: if value is None: # Need to force instantiation so we can reliably # tell if we get the same exception back value = t() try: # self.gen.throw(t, value, traceback) # raise RuntimeError("generator didn't stop after throw()") raise t(value) except StopIteration, exc: # Suppress the exception *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration # raised inside the "with" statement from being suppressed return exc is not value except: # only re-raise if it's *not* the exception that was # passed to throw(), because __exit__() must not raise # an exception unless __exit__() itself failed. But throw() # has to raise the exception to signal propagation, so this # fixes the impedance mismatch between the throw() protocol # and the __exit__() protocol. # if sys.exc_info()[1] is not value: raise def contextmanager(func): """@contextmanager decorator. Typical usage: @contextmanager def some_generator(): try: yield finally: This makes this: with some_generator() as : equivalent to this: try: = finally: """ @wraps(func) def helper(*args, **kwds): return GeneratorContextManager(func(*args, **kwds)) return helper @contextmanager def nested(*managers): """Combine multiple context managers into a single nested context manager. This function has been deprecated in favour of the multiple manager form of the with statement. The one advantage of this function over the multiple manager form of the with statement is that argument unpacking allows it to be used with a variable number of context managers as follows: with nested(*managers): do_something() """ warn("With-statements now directly support multiple context managers", DeprecationWarning, 3) exits = [] vars = [] exc = (None, None, None) try: for mgr in managers: exit = mgr.__exit__ enter = mgr.__enter__ vars.append(enter()) exits.append(exit) yield vars except: exc = sys.exc_info() finally: while exits: exit = exits.pop() try: if exit(*exc): exc = (None, None, None) except: exc = sys.exc_info() if exc != (None, None, None): # Don't rely on sys.exc_info() still containing # the right information. Another exception may # have been raised and caught by an exit method raise exc[0], exc[1], exc[2] class closing(object): """Context to automatically close something at the end of a block. Code like this: with closing(.open()) as f: is equivalent to this: f = .open() try: finally: f.close() """ def __init__(self, thing): self.thing = thing def __enter__(self): return self.thing def __exit__(self, *exc_info): self.thing.close() ================================================ FILE: third_party/stdlib/copy.py ================================================ """Generic (shallow and deep) copying operations. Interface summary: import copy x = copy.copy(y) # make a shallow copy of y x = copy.deepcopy(y) # make a deep copy of y For module specific errors, copy.Error is raised. The difference between shallow and deep copying is only relevant for compound objects (objects that contain other objects, like lists or class instances). - A shallow copy constructs a new compound object and then (to the extent possible) inserts *the same objects* into it that the original contains. - A deep copy constructs a new compound object and then, recursively, inserts *copies* into it of the objects found in the original. Two problems often exist with deep copy operations that don't exist with shallow copy operations: a) recursive objects (compound objects that, directly or indirectly, contain a reference to themselves) may cause a recursive loop b) because deep copy copies *everything* it may copy too much, e.g. administrative data structures that should be shared even between copies Python's deep copy operation avoids these problems by: a) keeping a table of objects already copied during the current copying pass b) letting user-defined classes override the copying operation or the set of components copied This version does not copy types like module, class, function, method, nor stack trace, stack frame, nor file, socket, window, nor array, nor any similar types. Classes can use the same interfaces to control copying that they use to control pickling: they can define methods called __getinitargs__(), __getstate__() and __setstate__(). See the documentation for module "pickle" for information on these methods. """ from '__go__/grumpy' import WeakRefType import types #from copy_reg import dispatch_table import copy_reg dispatch_table = copy_reg.dispatch_table # Import directly from grumpy to avoid circular imports with weakref module. class Error(Exception): pass error = Error # backward compatibility #try: # from org.python.core import PyStringMap #except ImportError: PyStringMap = None __all__ = ["Error", "copy", "deepcopy"] def copy(x): """Shallow copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) copier = getattr(cls, "__copy__", None) if copier: return copier(x) reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(2) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error("un(shallow)copyable object of type %s" % cls) return _reconstruct(x, rv, 0) _copy_dispatch = d = {} def _copy_immutable(x): return x for t in (type(None), int, float, bool, str, tuple, long, type, xrange, frozenset, #types.ClassType types.BuiltinFunctionType, # type(Ellipsis), types.FunctionType, WeakRefType): d[t] = _copy_immutable for name in ("ComplexType", "UnicodeType", "CodeType"): t = getattr(types, name, None) if t is not None: d[t] = _copy_immutable def _copy_with_constructor(x): return type(x)(x) for t in (list, dict): # , set d[t] = _copy_with_constructor def _copy_with_copy_method(x): return x.copy() if PyStringMap is not None: d[PyStringMap] = _copy_with_copy_method #def _copy_inst(x): # if hasattr(x, '__copy__'): # return x.__copy__() # if hasattr(x, '__getinitargs__'): # args = x.__getinitargs__() # y = x.__class__(*args) # else: # y = _EmptyClass() # y.__class__ = x.__class__ # if hasattr(x, '__getstate__'): # state = x.__getstate__() # else: # state = x.__dict__ # if hasattr(y, '__setstate__'): # y.__setstate__(state) # else: # y.__dict__.update(state) # return y #d[types.InstanceType] = _copy_inst del d def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ if memo is None: memo = {} d = id(x) y = memo.get(d, _nil) if y is not _nil: return y cls = type(x) copier = _deepcopy_dispatch.get(cls) if copier: y = copier(x, memo) else: try: issc = issubclass(cls, type) except TypeError: # cls is not a class (old Boost; see SF #502085) issc = 0 if issc: y = _deepcopy_atomic(x, memo) else: copier = getattr(x, "__deepcopy__", None) if copier: y = copier(memo) else: reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(2) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error( "un(deep)copyable object of type %s" % cls) y = _reconstruct(x, rv, 1, memo) memo[d] = y _keep_alive(x, memo) # Make sure x lives at least as long as d return y _deepcopy_dispatch = d = {} def _deepcopy_atomic(x, memo): return x d[type(None)] = _deepcopy_atomic #d[type(Ellipsis)] = _deepcopy_atomic d[int] = _deepcopy_atomic d[long] = _deepcopy_atomic d[float] = _deepcopy_atomic d[bool] = _deepcopy_atomic try: d[complex] = _deepcopy_atomic except NameError: pass d[str] = _deepcopy_atomic try: d[unicode] = _deepcopy_atomic except NameError: pass try: d[types.CodeType] = _deepcopy_atomic except AttributeError: pass d[type] = _deepcopy_atomic d[xrange] = _deepcopy_atomic #d[types.ClassType] = _deepcopy_atomic d[types.BuiltinFunctionType] = _deepcopy_atomic d[types.FunctionType] = _deepcopy_atomic d[WeakRefType] = _deepcopy_atomic def _deepcopy_list(x, memo): y = [] memo[id(x)] = y for a in x: y.append(deepcopy(a, memo)) return y d[list] = _deepcopy_list def _deepcopy_tuple(x, memo): y = [] for a in x: y.append(deepcopy(a, memo)) d = id(x) try: return memo[d] except KeyError: pass for i in range(len(x)): if x[i] is not y[i]: y = tuple(y) break else: y = x memo[d] = y return y d[tuple] = _deepcopy_tuple def _deepcopy_dict(x, memo): y = {} memo[id(x)] = y for key, value in x.iteritems(): y[deepcopy(key, memo)] = deepcopy(value, memo) return y d[dict] = _deepcopy_dict if PyStringMap is not None: d[PyStringMap] = _deepcopy_dict def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.im_func, deepcopy(x.im_self, memo), x.im_class) _deepcopy_dispatch[types.MethodType] = _deepcopy_method def _keep_alive(x, memo): """Keeps a reference to the object x in the memo. Because we remember objects by their id, we have to assure that possibly temporary objects are kept alive by referencing them. We store a reference at the id of the memo, which should normally not be used unless someone tries to deepcopy the memo itself... """ try: memo[id(memo)].append(x) except KeyError: # aha, this is the first one :-) memo[id(memo)]=[x] #def _deepcopy_inst(x, memo): # if hasattr(x, '__deepcopy__'): # return x.__deepcopy__(memo) # if hasattr(x, '__getinitargs__'): # args = x.__getinitargs__() # args = deepcopy(args, memo) # y = x.__class__(*args) # else: # y = _EmptyClass() # y.__class__ = x.__class__ # memo[id(x)] = y # if hasattr(x, '__getstate__'): # state = x.__getstate__() # else: # state = x.__dict__ # state = deepcopy(state, memo) # if hasattr(y, '__setstate__'): # y.__setstate__(state) # else: # y.__dict__.update(state) # return y #d[types.InstanceType] = _deepcopy_inst def _reconstruct(x, info, deep, memo=None): if isinstance(info, str): return x assert isinstance(info, tuple) if memo is None: memo = {} n = len(info) assert n in (2, 3, 4, 5) callable, args = info[:2] if n > 2: state = info[2] else: state = None if n > 3: listiter = info[3] else: listiter = None if n > 4: dictiter = info[4] else: dictiter = None if deep: args = deepcopy(args, memo) y = callable(*args) memo[id(x)] = y if state is not None: if deep: state = deepcopy(state, memo) if hasattr(y, '__setstate__'): y.__setstate__(state) else: if isinstance(state, tuple) and len(state) == 2: state, slotstate = state else: slotstate = None if state is not None: y.__dict__.update(state) if slotstate is not None: for key, value in slotstate.iteritems(): setattr(y, key, value) if listiter is not None: for item in listiter: if deep: item = deepcopy(item, memo) y.append(item) if dictiter is not None: for key, value in dictiter: if deep: key = deepcopy(key, memo) value = deepcopy(value, memo) y[key] = value return y del d del types # Helper for instance creation without calling __init__ #class _EmptyClass: # pass def _test(): #l = [None, 1, 2L, 3.14, 'xyzzy', (1, 2L), [3.14, 'abc'], l = [None, 1, 3.14, 'xyzzy', (1,), [3.14, 'abc'], {'abc': 'ABC'}, (), [], {}] l1 = copy(l) print l1==l l1 = map(copy, l) print l1==l l1 = deepcopy(l) print l1==l class C: def __init__(self, arg=None): self.a = 1 self.arg = arg if __name__ == '__main__': import sys file = sys.argv[0] else: file = __file__ self.fp = open(file) self.fp.close() def __getstate__(self): return {'a': self.a, 'arg': self.arg} def __setstate__(self, state): for key, value in state.iteritems(): setattr(self, key, value) def __deepcopy__(self, memo=None): new = self.__class__(deepcopy(self.arg, memo)) new.a = self.a return new c = C('argument sketch') l.append(c) l2 = copy(l) print l == l2 print l print l2 l2 = deepcopy(l) print l == l2 print l print l2 l.append({l[1]: l, 'xyz': l[2]}) l3 = copy(l) import repr print map(repr.repr, l) print map(repr.repr, l1) print map(repr.repr, l2) print map(repr.repr, l3) l3 = deepcopy(l) import repr print map(repr.repr, l) print map(repr.repr, l1) print map(repr.repr, l2) print map(repr.repr, l3) class odict(dict): def __init__(self, d = {}): self.a = 99 dict.__init__(self, d) def __setitem__(self, k, i): dict.__setitem__(self, k, i) self.a o = odict({"A" : "B"}) x = deepcopy(o) print(o, x) if __name__ == '__main__': _test() ================================================ FILE: third_party/stdlib/copy_reg.py ================================================ """Helper to provide extensibility for pickle/cPickle. This is only useful to add pickle support for extension types defined in C, not for instances of user-defined classes. """ #from types import ClassType as _ClassType __all__ = ["pickle", "constructor", "add_extension", "remove_extension", "clear_extension_cache"] dispatch_table = {} def pickle(ob_type, pickle_function, constructor_ob=None): # if type(ob_type) is _ClassType: # raise TypeError("copy_reg is not intended for use with classes") if not hasattr(pickle_function, '__call__'): raise TypeError("reduction functions must be callable") dispatch_table[ob_type] = pickle_function # The constructor_ob function is a vestige of safe for unpickling. # There is no reason for the caller to pass it anymore. if constructor_ob is not None: constructor(constructor_ob) def constructor(object): if not hasattr(object, '__call__'): raise TypeError("constructors must be callable") # Example: provide pickling support for complex numbers. try: complex except NameError: pass else: def pickle_complex(c): return complex, (c.real, c.imag) pickle(complex, pickle_complex, complex) # Support for pickling new-style objects def _reconstructor(cls, base, state): if base is object: obj = object.__new__(cls) else: obj = base.__new__(cls, state) if base.__init__ != object.__init__: base.__init__(obj, state) return obj _HEAPTYPE = 1<<9 # Python code for object.__reduce_ex__ for protocols 0 and 1 def _reduce_ex(self, proto): assert proto < 2 for base in self.__class__.__mro__: if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: break else: base = object # not really reachable if base is object: state = None else: if base is self.__class__: raise TypeError, "can't pickle %s objects" % base.__name__ state = base(self) args = (self.__class__, base, state) try: getstate = self.__getstate__ except AttributeError: if getattr(self, "__slots__", None): raise TypeError("a class that defines __slots__ without " "defining __getstate__ cannot be pickled") try: dict = self.__dict__ except AttributeError: dict = None else: dict = getstate() if dict: return _reconstructor, args, dict else: return _reconstructor, args # Helper for __reduce_ex__ protocol 2 def __newobj__(cls, *args): return cls.__new__(cls, *args) def _slotnames(cls): """Return a list of slot names for a given class. This needs to find slots defined by the class and its bases, so we can't simply return the __slots__ attribute. We must walk down the Method Resolution Order and concatenate the __slots__ of each class found there. (This assumes classes don't modify their __slots__ attribute to misrepresent their slots after the class is defined.) """ # Get the value from a cache in the class if possible names = cls.__dict__.get("__slotnames__") if names is not None: return names # Not cached -- calculate the value names = [] if not hasattr(cls, "__slots__"): # This class has no slots pass else: # Slots found -- gather slot names from all base classes for c in cls.__mro__: if "__slots__" in c.__dict__: slots = c.__dict__['__slots__'] # if class has a single slot, it can be given as a string if isinstance(slots, basestring): slots = (slots,) for name in slots: # special descriptors if name in ("__dict__", "__weakref__"): continue # mangled names elif name.startswith('__') and not name.endswith('__'): names.append('_%s%s' % (c.__name__, name)) else: names.append(name) # Cache the outcome in the class if at all possible try: cls.__slotnames__ = names except: pass # But don't die if we can't return names # A registry of extension codes. This is an ad-hoc compression # mechanism. Whenever a global reference to , is about # to be pickled, the (, ) tuple is looked up here to see # if it is a registered extension code for it. Extension codes are # universal, so that the meaning of a pickle does not depend on # context. (There are also some codes reserved for local use that # don't have this restriction.) Codes are positive ints; 0 is # reserved. _extension_registry = {} # key -> code _inverted_registry = {} # code -> key _extension_cache = {} # code -> object # Don't ever rebind those names: cPickle grabs a reference to them when # it's initialized, and won't see a rebinding. def add_extension(module, name, code): """Register an extension code.""" code = int(code) if not 1 <= code <= 0x7fffffff: raise ValueError, "code out of range" key = (module, name) if (_extension_registry.get(key) == code and _inverted_registry.get(code) == key): return # Redundant registrations are benign if key in _extension_registry: raise ValueError("key %s is already registered with code %s" % (key, _extension_registry[key])) if code in _inverted_registry: raise ValueError("code %s is already in use for key %s" % (code, _inverted_registry[code])) _extension_registry[key] = code _inverted_registry[code] = key def remove_extension(module, name, code): """Unregister an extension code. For testing only.""" key = (module, name) if (_extension_registry.get(key) != code or _inverted_registry.get(code) != key): raise ValueError("key %s is not registered with code %s" % (key, code)) del _extension_registry[key] del _inverted_registry[code] if code in _extension_cache: del _extension_cache[code] def clear_extension_cache(): _extension_cache.clear() # Standard extension code assignments # Reserved ranges # First Last Count Purpose # 1 127 127 Reserved for Python standard library # 128 191 64 Reserved for Zope # 192 239 48 Reserved for 3rd parties # 240 255 16 Reserved for private use (will never be assigned) # 256 Inf Inf Reserved for future assignment # Extension codes are assigned by the Python Software Foundation. ================================================ FILE: third_party/stdlib/csv.py ================================================ """ csv.py - read/write/investigate CSV files """ import re import functools reduce = functools.reduce # from functools import reduce # TODO: Support from foo import * syntax. import _csv for name in _csv.__all__: globals()[name] = getattr(_csv, name) # from _csv import Error, __version__, writer, reader, register_dialect, \ # unregister_dialect, get_dialect, list_dialects, \ # field_size_limit, \ # QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \ # __doc__ # from _csv import Dialect as _Dialect _Dialect = _csv.Dialect import StringIO as _StringIO StringIO = _StringIO.StringIO # try: # from cStringIO import StringIO # except ImportError: # from StringIO import StringIO __all__ = [ "QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", "Error", "Dialect", "__doc__", "excel", "excel_tab", "field_size_limit", "reader", "writer", "register_dialect", "get_dialect", "list_dialects", "Sniffer", "unregister_dialect", "__version__", "DictReader", "DictWriter" ] class Dialect(object): """Describe an Excel dialect. This must be subclassed (see csv.excel). Valid attributes are: delimiter, quotechar, escapechar, doublequote, skipinitialspace, lineterminator, quoting. """ _name = "" _valid = False # placeholders delimiter = None quotechar = None escapechar = None doublequote = None skipinitialspace = None lineterminator = None quoting = None def __init__(self): if self.__class__ != Dialect: self._valid = True self._validate() def _validate(self): try: _Dialect(self) except TypeError, e: # We do this for compatibility with py2.3 raise Error(str(e)) class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("excel", excel) class excel_tab(excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = '\t' register_dialect("excel-tab", excel_tab) class DictReader(object): def __init__(self, f, fieldnames=None, restkey=None, restval=None, dialect="excel", *args, **kwds): self._fieldnames = fieldnames # list of keys for the dict self.restkey = restkey # key to catch long rows self.restval = restval # default value for short rows self.reader = reader(f, dialect, *args, **kwds) self.dialect = dialect self.line_num = 0 def __iter__(self): return self # @property def fieldnames(self): if self._fieldnames is None: try: self._fieldnames = self.reader.next() except StopIteration: pass self.line_num = self.reader.line_num return self._fieldnames fieldnames = property(fieldnames) # Issue 20004: Because DictReader is a classic class, this setter is # ignored. At this point in 2.7's lifecycle, it is too late to change the # base class for fear of breaking working code. If you want to change # fieldnames without overwriting the getter, set _fieldnames directly. @fieldnames.setter def fieldnames(self, value): self._fieldnames = value def next(self): if self.line_num == 0: # Used only for its side effect. self.fieldnames row = self.reader.next() self.line_num = self.reader.line_num # unlike the basic reader, we prefer not to return blanks, # because we will typically wind up with a dict full of None # values while row == []: row = self.reader.next() d = dict(zip(self.fieldnames, row)) lf = len(self.fieldnames) lr = len(row) if lf < lr: d[self.restkey] = row[lf:] elif lf > lr: for key in self.fieldnames[lr:]: d[key] = self.restval return d class DictWriter(object): def __init__(self, f, fieldnames, restval="", extrasaction="raise", dialect="excel", *args, **kwds): self.fieldnames = fieldnames # list of keys for the dict self.restval = restval # for writing short dicts if extrasaction.lower() not in ("raise", "ignore"): raise ValueError, \ ("extrasaction (%s) must be 'raise' or 'ignore'" % extrasaction) self.extrasaction = extrasaction self.writer = writer(f, dialect, *args, **kwds) def writeheader(self): header = dict(zip(self.fieldnames, self.fieldnames)) self.writerow(header) def _dict_to_list(self, rowdict): if self.extrasaction == "raise": wrong_fields = [k for k in rowdict if k not in self.fieldnames] if wrong_fields: raise ValueError("dict contains fields not in fieldnames: " + ", ".join([repr(x) for x in wrong_fields])) return [rowdict.get(key, self.restval) for key in self.fieldnames] def writerow(self, rowdict): return self.writer.writerow(self._dict_to_list(rowdict)) def writerows(self, rowdicts): rows = [] for rowdict in rowdicts: rows.append(self._dict_to_list(rowdict)) return self.writer.writerows(rows) # Guard Sniffer's type checking against builds that exclude complex() # try: # complex # except NameError: # complex = float complex = float class Sniffer(object): ''' "Sniffs" the format of a CSV file (i.e. delimiter, quotechar) Returns a Dialect object. ''' def __init__(self): # in case there is more than one possible delimiter self.preferred = [',', '\t', ';', ' ', ':'] def sniff(self, sample, delimiters=None): """ Returns a dialect (or None) corresponding to the sample """ quotechar, doublequote, delimiter, skipinitialspace = \ self._guess_quote_and_delimiter(sample, delimiters) if not delimiter: delimiter, skipinitialspace = self._guess_delimiter(sample, delimiters) if not delimiter: raise Error, "Could not determine delimiter" class dialect(Dialect): _name = "sniffed" lineterminator = '\r\n' quoting = QUOTE_MINIMAL # escapechar = '' dialect.doublequote = doublequote dialect.delimiter = delimiter # _csv.reader won't accept a quotechar of '' dialect.quotechar = quotechar or '"' dialect.skipinitialspace = skipinitialspace return dialect def _guess_quote_and_delimiter(self, data, delimiters): """ Looks for text enclosed between two identical quotes (the probable quotechar) which are preceded and followed by the same character (the probable delimiter). For example: ,'some text', The quote with the most wins, same with the delimiter. If there is no quotechar the delimiter can't be determined this way. """ matches = [] for restr in ('(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", '(?:^|\n)(?P["\']).*?(?P=quote)(?P[^\w\n"\'])(?P ?)', # ".*?", '(?P>[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?:$|\n)', # ,".*?" '(?:^|\n)(?P["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space) regexp = re.compile(restr, re.DOTALL | re.MULTILINE) matches = regexp.findall(data) if matches: break if not matches: # (quotechar, doublequote, delimiter, skipinitialspace) return ('', False, None, 0) quotes = {} delims = {} spaces = 0 for m in matches: n = regexp.groupindex['quote'] - 1 key = m[n] if key: quotes[key] = quotes.get(key, 0) + 1 try: n = regexp.groupindex['delim'] - 1 key = m[n] except KeyError: continue if key and (delimiters is None or key in delimiters): delims[key] = delims.get(key, 0) + 1 try: n = regexp.groupindex['space'] - 1 except KeyError: continue if m[n]: spaces += 1 quotechar = reduce(lambda a, b, quotes = quotes: (quotes[a] > quotes[b]) and a or b, quotes.keys()) if delims: delim = reduce(lambda a, b, delims = delims: (delims[a] > delims[b]) and a or b, delims.keys()) skipinitialspace = delims[delim] == spaces if delim == '\n': # most likely a file with a single column delim = '' else: # there is *no* delimiter, it's a single column of quoted data delim = '' skipinitialspace = 0 # if we see an extra quote between delimiters, we've got a # double quoted format dq_regexp = re.compile( r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) if dq_regexp.search(data): doublequote = True else: doublequote = False return (quotechar, doublequote, delim, skipinitialspace) def _guess_delimiter(self, data, delimiters): """ The delimiter /should/ occur the same number of times on each row. However, due to malformed data, it may not. We don't want an all or nothing approach, so we allow for small variations in this number. 1) build a table of the frequency of each character on every line. 2) build a table of frequencies of this frequency (meta-frequency?), e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows, 7 times in 2 rows' 3) use the mode of the meta-frequency to determine the /expected/ frequency for that character 4) find out how often the character actually meets that goal 5) the character that best meets its goal is the delimiter For performance reasons, the data is evaluated in chunks, so it can try and evaluate the smallest portion of the data possible, evaluating additional chunks as necessary. """ data = filter(None, data.split('\n')) ascii = [chr(c) for c in range(127)] # 7-bit ASCII # build frequency tables chunkLength = min(10, len(data)) iteration = 0 charFrequency = {} modes = {} delims = {} start, end = 0, min(chunkLength, len(data)) while start < len(data): iteration += 1 for line in data[start:end]: for char in ascii: metaFrequency = charFrequency.get(char, {}) # must count even if frequency is 0 freq = line.count(char) # value is the mode metaFrequency[freq] = metaFrequency.get(freq, 0) + 1 charFrequency[char] = metaFrequency for char in charFrequency.keys(): items = charFrequency[char].items() if len(items) == 1 and items[0][0] == 0: continue # get the mode of the frequencies if len(items) > 1: modes[char] = reduce(lambda a, b: a[1] > b[1] and a or b, items) # adjust the mode - subtract the sum of all # other frequencies items.remove(modes[char]) modes[char] = (modes[char][0], modes[char][1] - reduce(lambda a, b: (0, a[1] + b[1]), items)[1]) else: modes[char] = items[0] # build a list of possible delimiters modeList = modes.items() total = float(chunkLength * iteration) # (rows of consistent data) / (number of rows) = 100% consistency = 1.0 # minimum consistency threshold threshold = 0.9 while len(delims) == 0 and consistency >= threshold: for k, v in modeList: if v[0] > 0 and v[1] > 0: if ((v[1]/total) >= consistency and (delimiters is None or k in delimiters)): delims[k] = v consistency -= 0.01 if len(delims) == 1: delim = delims.keys()[0] skipinitialspace = (data[0].count(delim) == data[0].count("%c " % delim)) return (delim, skipinitialspace) # analyze another chunkLength lines start = end end += chunkLength if not delims: return ('', 0) # if there's more than one, fall back to a 'preferred' list if len(delims) > 1: for d in self.preferred: if d in delims.keys(): skipinitialspace = (data[0].count(d) == data[0].count("%c " % d)) return (d, skipinitialspace) # nothing else indicates a preference, pick the character that # dominates(?) items = [(v,k) for (k,v) in delims.items()] items.sort() delim = items[-1][1] skipinitialspace = (data[0].count(delim) == data[0].count("%c " % delim)) return (delim, skipinitialspace) def has_header(self, sample): # Creates a dictionary of types of data in each column. If any # column is of a single type (say, integers), *except* for the first # row, then the first row is presumed to be labels. If the type # can't be determined, it is assumed to be a string in which case # the length of the string is the determining factor: if all of the # rows except for the first are the same length, it's a header. # Finally, a 'vote' is taken at the end for each column, adding or # subtracting from the likelihood of the first row being a header. rdr = reader(StringIO(sample), self.sniff(sample)) header = rdr.next() # assume first row is header columns = len(header) columnTypes = {} for i in range(columns): columnTypes[i] = None checked = 0 for row in rdr: # arbitrary number of rows to check, to keep it sane if checked > 20: break checked += 1 if len(row) != columns: continue # skip rows that have irregular number of columns for col in columnTypes.keys(): for thisType in [int, long, float, complex]: try: thisType(row[col]) break except (ValueError, OverflowError): pass else: # fallback to length of string thisType = len(row[col]) # treat longs as ints if thisType == long: thisType = int if thisType != columnTypes[col]: if columnTypes[col] is None: # add new column type columnTypes[col] = thisType else: # type is inconsistent, remove column from # consideration del columnTypes[col] # finally, compare results against first row and "vote" # on whether it's a header hasHeader = 0 for col, colType in columnTypes.items(): if type(colType) == type(0): # it's a length if len(header[col]) != colType: hasHeader += 1 else: hasHeader -= 1 else: # attempt typecast try: colType(header[col]) except (ValueError, TypeError): hasHeader += 1 else: hasHeader -= 1 return hasHeader > 0 ================================================ FILE: third_party/stdlib/difflib.py ================================================ """ Module difflib -- helpers for computing deltas between objects. Function get_close_matches(word, possibilities, n=3, cutoff=0.6): Use SequenceMatcher to return list of the best "good enough" matches. Function context_diff(a, b): For two lists of strings, return a delta in context diff format. Function ndiff(a, b): Return a delta: the difference between `a` and `b` (lists of strings). Function restore(delta, which): Return one of the two sequences that generated an ndiff delta. Function unified_diff(a, b): For two lists of strings, return a delta in unified diff format. Class SequenceMatcher: A flexible class for comparing pairs of sequences of any type. Class Differ: For producing human-readable deltas from sequences of lines of text. Class HtmlDiff: For producing HTML side by side comparison with change highlights. """ __all__ = ['get_close_matches', 'ndiff', 'restore', 'SequenceMatcher', 'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff', 'unified_diff', 'HtmlDiff', 'Match'] import heapq # from collections import namedtuple as _namedtuple # from functools import reduce import functools reduce = functools.reduce import operator _itemgetter = operator.itemgetter _property = property _tuple = tuple def setdefault(d, k, default=None): if k not in d: d[k] = default return d[k] # Match = _namedtuple('Match', 'a b size') class Match(tuple): 'Match(a, b, size)' __slots__ = () _fields = ('a', 'b', 'size') def __new__(_cls, a, b, size): 'Create new instance of Match(a, b, size)' return _tuple.__new__(_cls, (a, b, size)) # @classmethod def _make(cls, iterable, new=tuple.__new__, len=len): 'Make a new Match object from a sequence or iterable' result = new(cls, iterable) if len(result) != 3: raise TypeError('Expected 3 arguments, got %d' % len(result)) return result _make = classmethod(_make) def __repr__(self): 'Return a nicely formatted representation string' return 'Match(a=%r, b=%r, size=%r)' % self def _asdict(self): 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) def _replace(_self, **kwds): 'Return a new Match object replacing specified fields with new values' result = _self._make(map(kwds.pop, ('a', 'b', 'size'), _self)) if kwds: raise ValueError('Got unexpected field names: %r' % kwds.keys()) return result def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) __dict__ = _property(_asdict) def __getstate__(self): 'Exclude the OrderedDict from pickling' pass a = _property(_itemgetter(0), doc='Alias for field number 0') b = _property(_itemgetter(1), doc='Alias for field number 1') size = _property(_itemgetter(2), doc='Alias for field number 2') def _calculate_ratio(matches, length): if length: return 2.0 * matches / length return 1.0 class SequenceMatcher(object): """ SequenceMatcher is a flexible class for comparing pairs of sequences of any type, so long as the sequence elements are hashable. The basic algorithm predates, and is a little fancier than, an algorithm published in the late 1980's by Ratcliff and Obershelp under the hyperbolic name "gestalt pattern matching". The basic idea is to find the longest contiguous matching subsequence that contains no "junk" elements (R-O doesn't address junk). The same idea is then applied recursively to the pieces of the sequences to the left and to the right of the matching subsequence. This does not yield minimal edit sequences, but does tend to yield matches that "look right" to people. SequenceMatcher tries to compute a "human-friendly diff" between two sequences. Unlike e.g. UNIX(tm) diff, the fundamental notion is the longest *contiguous* & junk-free matching subsequence. That's what catches peoples' eyes. The Windows(tm) windiff has another interesting notion, pairing up elements that appear uniquely in each sequence. That, and the method here, appear to yield more intuitive difference reports than does diff. This method appears to be the least vulnerable to synching up on blocks of "junk lines", though (like blank lines in ordinary text files, or maybe "

" lines in HTML files). That may be because this is the only method of the 3 that has a *concept* of "junk" . Example, comparing two strings, and considering blanks to be "junk": >>> s = SequenceMatcher(lambda x: x == " ", ... "private Thread currentThread;", ... "private volatile Thread currentThread;") >>> .ratio() returns a float in [0, 1], measuring the "similarity" of the sequences. As a rule of thumb, a .ratio() value over 0.6 means the sequences are close matches: >>> print round(s.ratio(), 3) 0.866 >>> If you're only interested in where the sequences match, .get_matching_blocks() is handy: >>> for block in s.get_matching_blocks(): ... print "a[%d] and b[%d] match for %d elements" % block a[0] and b[0] match for 8 elements a[8] and b[17] match for 21 elements a[29] and b[38] match for 0 elements Note that the last tuple returned by .get_matching_blocks() is always a dummy, (len(a), len(b), 0), and this is the only case in which the last tuple element (number of elements matched) is 0. If you want to know how to change the first sequence into the second, use .get_opcodes(): >>> for opcode in s.get_opcodes(): ... print "%6s a[%d:%d] b[%d:%d]" % opcode equal a[0:8] b[0:8] insert a[8:8] b[8:17] equal a[8:29] b[17:38] See the Differ class for a fancy human-friendly file differencer, which uses SequenceMatcher both to compare sequences of lines, and to compare sequences of characters within similar (near-matching) lines. See also function get_close_matches() in this module, which shows how simple code building on SequenceMatcher can be used to do useful work. Timing: Basic R-O is cubic time worst case and quadratic time expected case. SequenceMatcher is quadratic time for the worst case and has expected-case behavior dependent in a complicated way on how many elements the sequences have in common; best case time is linear. Methods: __init__(isjunk=None, a='', b='') Construct a SequenceMatcher. set_seqs(a, b) Set the two sequences to be compared. set_seq1(a) Set the first sequence to be compared. set_seq2(b) Set the second sequence to be compared. find_longest_match(alo, ahi, blo, bhi) Find longest matching block in a[alo:ahi] and b[blo:bhi]. get_matching_blocks() Return list of triples describing matching subsequences. get_opcodes() Return list of 5-tuples describing how to turn a into b. ratio() Return a measure of the sequences' similarity (float in [0,1]). quick_ratio() Return an upper bound on .ratio() relatively quickly. real_quick_ratio() Return an upper bound on ratio() very quickly. """ def __init__(self, isjunk=None, a='', b='', autojunk=True): """Construct a SequenceMatcher. Optional arg isjunk is None (the default), or a one-argument function that takes a sequence element and returns true iff the element is junk. None is equivalent to passing "lambda x: 0", i.e. no elements are considered to be junk. For example, pass lambda x: x in " \\t" if you're comparing lines as sequences of characters, and don't want to synch up on blanks or hard tabs. Optional arg a is the first of two sequences to be compared. By default, an empty string. The elements of a must be hashable. See also .set_seqs() and .set_seq1(). Optional arg b is the second of two sequences to be compared. By default, an empty string. The elements of b must be hashable. See also .set_seqs() and .set_seq2(). Optional arg autojunk should be set to False to disable the "automatic junk heuristic" that treats popular elements as junk (see module documentation for more information). """ # Members: # a # first sequence # b # second sequence; differences are computed as "what do # we need to do to 'a' to change it into 'b'?" # b2j # for x in b, b2j[x] is a list of the indices (into b) # at which x appears; junk elements do not appear # fullbcount # for x in b, fullbcount[x] == the number of times x # appears in b; only materialized if really needed (used # only for computing quick_ratio()) # matching_blocks # a list of (i, j, k) triples, where a[i:i+k] == b[j:j+k]; # ascending & non-overlapping in i and in j; terminated by # a dummy (len(a), len(b), 0) sentinel # opcodes # a list of (tag, i1, i2, j1, j2) tuples, where tag is # one of # 'replace' a[i1:i2] should be replaced by b[j1:j2] # 'delete' a[i1:i2] should be deleted # 'insert' b[j1:j2] should be inserted # 'equal' a[i1:i2] == b[j1:j2] # isjunk # a user-supplied function taking a sequence element and # returning true iff the element is "junk" -- this has # subtle but helpful effects on the algorithm, which I'll # get around to writing up someday <0.9 wink>. # DON'T USE! Only __chain_b uses this. Use isbjunk. # isbjunk # for x in b, isbjunk(x) == isjunk(x) but much faster; # it's really the __contains__ method of a hidden dict. # DOES NOT WORK for x in a! # isbpopular # for x in b, isbpopular(x) is true iff b is reasonably long # (at least 200 elements) and x accounts for more than 1 + 1% of # its elements (when autojunk is enabled). # DOES NOT WORK for x in a! self.isjunk = isjunk self.a = self.b = None self.autojunk = autojunk self.set_seqs(a, b) def set_seqs(self, a, b): """Set the two sequences to be compared. >>> s = SequenceMatcher() >>> s.set_seqs("abcd", "bcde") >>> s.ratio() 0.75 """ self.set_seq1(a) self.set_seq2(b) def set_seq1(self, a): """Set the first sequence to be compared. The second sequence to be compared is not changed. >>> s = SequenceMatcher(None, "abcd", "bcde") >>> s.ratio() 0.75 >>> s.set_seq1("bcde") >>> s.ratio() 1.0 >>> SequenceMatcher computes and caches detailed information about the second sequence, so if you want to compare one sequence S against many sequences, use .set_seq2(S) once and call .set_seq1(x) repeatedly for each of the other sequences. See also set_seqs() and set_seq2(). """ if a is self.a: return self.a = a self.matching_blocks = self.opcodes = None def set_seq2(self, b): """Set the second sequence to be compared. The first sequence to be compared is not changed. >>> s = SequenceMatcher(None, "abcd", "bcde") >>> s.ratio() 0.75 >>> s.set_seq2("abcd") >>> s.ratio() 1.0 >>> SequenceMatcher computes and caches detailed information about the second sequence, so if you want to compare one sequence S against many sequences, use .set_seq2(S) once and call .set_seq1(x) repeatedly for each of the other sequences. See also set_seqs() and set_seq1(). """ if b is self.b: return self.b = b self.matching_blocks = self.opcodes = None self.fullbcount = None self.__chain_b() # For each element x in b, set b2j[x] to a list of the indices in # b where x appears; the indices are in increasing order; note that # the number of times x appears in b is len(b2j[x]) ... # when self.isjunk is defined, junk elements don't show up in this # map at all, which stops the central find_longest_match method # from starting any matching block at a junk element ... # also creates the fast isbjunk function ... # b2j also does not contain entries for "popular" elements, meaning # elements that account for more than 1 + 1% of the total elements, and # when the sequence is reasonably large (>= 200 elements); this can # be viewed as an adaptive notion of semi-junk, and yields an enormous # speedup when, e.g., comparing program files with hundreds of # instances of "return NULL;" ... # note that this is only called when b changes; so for cross-product # kinds of matches, it's best to call set_seq2 once, then set_seq1 # repeatedly def __chain_b(self): # Because isjunk is a user-defined (not C) function, and we test # for junk a LOT, it's important to minimize the number of calls. # Before the tricks described here, __chain_b was by far the most # time-consuming routine in the whole module! If anyone sees # Jim Roskind, thank him again for profile.py -- I never would # have guessed that. # The first trick is to build b2j ignoring the possibility # of junk. I.e., we don't call isjunk at all yet. Throwing # out the junk later is much cheaper than building b2j "right" # from the start. b = self.b self.b2j = b2j = {} for i, elt in enumerate(b): indices = setdefault(b2j, elt, []) # indices = b2j.setdefault(elt, []) indices.append(i) # Purge junk elements junk = set() isjunk = self.isjunk if isjunk: for elt in list(b2j.keys()): # using list() since b2j is modified if isjunk(elt): junk.add(elt) del b2j[elt] # Purge popular elements that are not junk popular = set() n = len(b) if self.autojunk and n >= 200: ntest = n // 100 + 1 for elt, idxs in list(b2j.items()): if len(idxs) > ntest: popular.add(elt) del b2j[elt] # Now for x in b, isjunk(x) == x in junk, but the latter is much faster. # Sicne the number of *unique* junk elements is probably small, the # memory burden of keeping this set alive is likely trivial compared to # the size of b2j. self.isbjunk = junk.__contains__ self.isbpopular = popular.__contains__ def find_longest_match(self, alo, ahi, blo, bhi): """Find longest matching block in a[alo:ahi] and b[blo:bhi]. If isjunk is not defined: Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where alo <= i <= i+k <= ahi blo <= j <= j+k <= bhi and for all (i',j',k') meeting those conditions, k >= k' i <= i' and if i == i', j <= j' In other words, of all maximal matching blocks, return one that starts earliest in a, and of all those maximal matching blocks that start earliest in a, return the one that starts earliest in b. >>> s = SequenceMatcher(None, " abcd", "abcd abcd") >>> s.find_longest_match(0, 5, 0, 9) Match(a=0, b=4, size=5) If isjunk is defined, first the longest matching block is determined as above, but with the additional restriction that no junk element appears in the block. Then that block is extended as far as possible by matching (only) junk elements on both sides. So the resulting block never matches on junk except as identical junk happens to be adjacent to an "interesting" match. Here's the same example as before, but considering blanks to be junk. That prevents " abcd" from matching the " abcd" at the tail end of the second sequence directly. Instead only the "abcd" can match, and matches the leftmost "abcd" in the second sequence: >>> s = SequenceMatcher(lambda x: x==" ", " abcd", "abcd abcd") >>> s.find_longest_match(0, 5, 0, 9) Match(a=1, b=0, size=4) If no blocks match, return (alo, blo, 0). >>> s = SequenceMatcher(None, "ab", "c") >>> s.find_longest_match(0, 2, 0, 1) Match(a=0, b=0, size=0) """ # CAUTION: stripping common prefix or suffix would be incorrect. # E.g., # ab # acab # Longest matching block is "ab", but if common prefix is # stripped, it's "a" (tied with "b"). UNIX(tm) diff does so # strip, so ends up claiming that ab is changed to acab by # inserting "ca" in the middle. That's minimal but unintuitive: # "it's obvious" that someone inserted "ac" at the front. # Windiff ends up at the same place as diff, but by pairing up # the unique 'b's and then matching the first two 'a's. a, b, b2j, isbjunk = self.a, self.b, self.b2j, self.isbjunk besti, bestj, bestsize = alo, blo, 0 # find longest junk-free match # during an iteration of the loop, j2len[j] = length of longest # junk-free match ending with a[i-1] and b[j] j2len = {} nothing = [] for i in xrange(alo, ahi): # look at all instances of a[i] in b; note that because # b2j has no junk keys, the loop is skipped if a[i] is junk j2lenget = j2len.get newj2len = {} for j in b2j.get(a[i], nothing): # a[i] matches b[j] if j < blo: continue if j >= bhi: break k = newj2len[j] = j2lenget(j-1, 0) + 1 if k > bestsize: besti, bestj, bestsize = i-k+1, j-k+1, k j2len = newj2len # Extend the best by non-junk elements on each end. In particular, # "popular" non-junk elements aren't in b2j, which greatly speeds # the inner loop above, but also means "the best" match so far # doesn't contain any junk *or* popular non-junk elements. while besti > alo and bestj > blo and \ not isbjunk(b[bestj-1]) and \ a[besti-1] == b[bestj-1]: besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 while besti+bestsize < ahi and bestj+bestsize < bhi and \ not isbjunk(b[bestj+bestsize]) and \ a[besti+bestsize] == b[bestj+bestsize]: bestsize += 1 # Now that we have a wholly interesting match (albeit possibly # empty!), we may as well suck up the matching junk on each # side of it too. Can't think of a good reason not to, and it # saves post-processing the (possibly considerable) expense of # figuring out what to do with it. In the case of an empty # interesting match, this is clearly the right thing to do, # because no other kind of match is possible in the regions. while besti > alo and bestj > blo and \ isbjunk(b[bestj-1]) and \ a[besti-1] == b[bestj-1]: besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 while besti+bestsize < ahi and bestj+bestsize < bhi and \ isbjunk(b[bestj+bestsize]) and \ a[besti+bestsize] == b[bestj+bestsize]: bestsize = bestsize + 1 return Match(besti, bestj, bestsize) def get_matching_blocks(self): """Return list of triples describing matching subsequences. Each triple is of the form (i, j, n), and means that a[i:i+n] == b[j:j+n]. The triples are monotonically increasing in i and in j. New in Python 2.5, it's also guaranteed that if (i, j, n) and (i', j', n') are adjacent triples in the list, and the second is not the last triple in the list, then i+n != i' or j+n != j'. IOW, adjacent triples never describe adjacent equal blocks. The last triple is a dummy, (len(a), len(b), 0), and is the only triple with n==0. >>> s = SequenceMatcher(None, "abxcd", "abcd") >>> s.get_matching_blocks() [Match(a=0, b=0, size=2), Match(a=3, b=2, size=2), Match(a=5, b=4, size=0)] """ if self.matching_blocks is not None: return self.matching_blocks la, lb = len(self.a), len(self.b) # This is most naturally expressed as a recursive algorithm, but # at least one user bumped into extreme use cases that exceeded # the recursion limit on their box. So, now we maintain a list # ('queue`) of blocks we still need to look at, and append partial # results to `matching_blocks` in a loop; the matches are sorted # at the end. queue = [(0, la, 0, lb)] matching_blocks = [] while queue: alo, ahi, blo, bhi = queue.pop() i, j, k = x = self.find_longest_match(alo, ahi, blo, bhi) # a[alo:i] vs b[blo:j] unknown # a[i:i+k] same as b[j:j+k] # a[i+k:ahi] vs b[j+k:bhi] unknown if k: # if k is 0, there was no matching block matching_blocks.append(x) if alo < i and blo < j: queue.append((alo, i, blo, j)) if i+k < ahi and j+k < bhi: queue.append((i+k, ahi, j+k, bhi)) matching_blocks.sort() # It's possible that we have adjacent equal blocks in the # matching_blocks list now. Starting with 2.5, this code was added # to collapse them. i1 = j1 = k1 = 0 non_adjacent = [] for i2, j2, k2 in matching_blocks: # Is this block adjacent to i1, j1, k1? if i1 + k1 == i2 and j1 + k1 == j2: # Yes, so collapse them -- this just increases the length of # the first block by the length of the second, and the first # block so lengthened remains the block to compare against. k1 += k2 else: # Not adjacent. Remember the first block (k1==0 means it's # the dummy we started with), and make the second block the # new block to compare against. if k1: non_adjacent.append((i1, j1, k1)) i1, j1, k1 = i2, j2, k2 if k1: non_adjacent.append((i1, j1, k1)) non_adjacent.append( (la, lb, 0) ) self.matching_blocks = map(Match._make, non_adjacent) return self.matching_blocks def get_opcodes(self): """Return list of 5-tuples describing how to turn a into b. Each tuple is of the form (tag, i1, i2, j1, j2). The first tuple has i1 == j1 == 0, and remaining tuples have i1 == the i2 from the tuple preceding it, and likewise for j1 == the previous j2. The tags are strings, with these meanings: 'replace': a[i1:i2] should be replaced by b[j1:j2] 'delete': a[i1:i2] should be deleted. Note that j1==j2 in this case. 'insert': b[j1:j2] should be inserted at a[i1:i1]. Note that i1==i2 in this case. 'equal': a[i1:i2] == b[j1:j2] >>> a = "qabxcd" >>> b = "abycdf" >>> s = SequenceMatcher(None, a, b) >>> for tag, i1, i2, j1, j2 in s.get_opcodes(): ... print ("%7s a[%d:%d] (%s) b[%d:%d] (%s)" % ... (tag, i1, i2, a[i1:i2], j1, j2, b[j1:j2])) delete a[0:1] (q) b[0:0] () equal a[1:3] (ab) b[0:2] (ab) replace a[3:4] (x) b[2:3] (y) equal a[4:6] (cd) b[3:5] (cd) insert a[6:6] () b[5:6] (f) """ if self.opcodes is not None: return self.opcodes i = j = 0 self.opcodes = answer = [] for ai, bj, size in self.get_matching_blocks(): # invariant: we've pumped out correct diffs to change # a[:i] into b[:j], and the next matching block is # a[ai:ai+size] == b[bj:bj+size]. So we need to pump # out a diff to change a[i:ai] into b[j:bj], pump out # the matching block, and move (i,j) beyond the match tag = '' if i < ai and j < bj: tag = 'replace' elif i < ai: tag = 'delete' elif j < bj: tag = 'insert' if tag: answer.append( (tag, i, ai, j, bj) ) i, j = ai+size, bj+size # the list of matching blocks is terminated by a # sentinel with size 0 if size: answer.append( ('equal', ai, i, bj, j) ) return answer def get_grouped_opcodes(self, n=3): """ Isolate change clusters by eliminating ranges with no changes. Return a generator of groups with up to n lines of context. Each group is in the same format as returned by get_opcodes(). >>> from pprint import pprint >>> a = map(str, range(1,40)) >>> b = a[:] >>> b[8:8] = ['i'] # Make an insertion >>> b[20] += 'x' # Make a replacement >>> b[23:28] = [] # Make a deletion >>> b[30] += 'y' # Make another replacement >>> pprint(list(SequenceMatcher(None,a,b).get_grouped_opcodes())) [[('equal', 5, 8, 5, 8), ('insert', 8, 8, 8, 9), ('equal', 8, 11, 9, 12)], [('equal', 16, 19, 17, 20), ('replace', 19, 20, 20, 21), ('equal', 20, 22, 21, 23), ('delete', 22, 27, 23, 23), ('equal', 27, 30, 23, 26)], [('equal', 31, 34, 27, 30), ('replace', 34, 35, 30, 31), ('equal', 35, 38, 31, 34)]] """ codes = self.get_opcodes() if not codes: codes = [("equal", 0, 1, 0, 1)] # Fixup leading and trailing groups if they show no changes. if codes[0][0] == 'equal': tag, i1, i2, j1, j2 = codes[0] codes[0] = tag, max(i1, i2-n), i2, max(j1, j2-n), j2 if codes[-1][0] == 'equal': tag, i1, i2, j1, j2 = codes[-1] codes[-1] = tag, i1, min(i2, i1+n), j1, min(j2, j1+n) nn = n + n group = [] for tag, i1, i2, j1, j2 in codes: # End the current group and start a new one whenever # there is a large range with no changes. if tag == 'equal' and i2-i1 > nn: group.append((tag, i1, min(i2, i1+n), j1, min(j2, j1+n))) yield group group = [] i1, j1 = max(i1, i2-n), max(j1, j2-n) group.append((tag, i1, i2, j1 ,j2)) if group and not (len(group)==1 and group[0][0] == 'equal'): yield group def ratio(self): """Return a measure of the sequences' similarity (float in [0,1]). Where T is the total number of elements in both sequences, and M is the number of matches, this is 2.0*M / T. Note that this is 1 if the sequences are identical, and 0 if they have nothing in common. .ratio() is expensive to compute if you haven't already computed .get_matching_blocks() or .get_opcodes(), in which case you may want to try .quick_ratio() or .real_quick_ratio() first to get an upper bound. >>> s = SequenceMatcher(None, "abcd", "bcde") >>> s.ratio() 0.75 >>> s.quick_ratio() 0.75 >>> s.real_quick_ratio() 1.0 """ matches = reduce(lambda sum, triple: sum + triple[-1], self.get_matching_blocks(), 0) return _calculate_ratio(matches, len(self.a) + len(self.b)) def quick_ratio(self): """Return an upper bound on ratio() relatively quickly. This isn't defined beyond that it is an upper bound on .ratio(), and is faster to compute. """ # viewing a and b as multisets, set matches to the cardinality # of their intersection; this counts the number of matches # without regard to order, so is clearly an upper bound if self.fullbcount is None: self.fullbcount = fullbcount = {} for elt in self.b: fullbcount[elt] = fullbcount.get(elt, 0) + 1 fullbcount = self.fullbcount # avail[x] is the number of times x appears in 'b' less the # number of times we've seen it in 'a' so far ... kinda avail = {} availhas, matches = avail.__contains__, 0 for elt in self.a: if availhas(elt): numb = avail[elt] else: numb = fullbcount.get(elt, 0) avail[elt] = numb - 1 if numb > 0: matches = matches + 1 return _calculate_ratio(matches, len(self.a) + len(self.b)) def real_quick_ratio(self): """Return an upper bound on ratio() very quickly. This isn't defined beyond that it is an upper bound on .ratio(), and is faster to compute than either .ratio() or .quick_ratio(). """ la, lb = len(self.a), len(self.b) # can't have more matches than the number of elements in the # shorter sequence return _calculate_ratio(min(la, lb), la + lb) def get_close_matches(word, possibilities, n=3, cutoff=0.6): """Use SequenceMatcher to return list of the best "good enough" matches. word is a sequence for which close matches are desired (typically a string). possibilities is a list of sequences against which to match word (typically a list of strings). Optional arg n (default 3) is the maximum number of close matches to return. n must be > 0. Optional arg cutoff (default 0.6) is a float in [0, 1]. Possibilities that don't score at least that similar to word are ignored. The best (no more than n) matches among the possibilities are returned in a list, sorted by similarity score, most similar first. >>> get_close_matches("appel", ["ape", "apple", "peach", "puppy"]) ['apple', 'ape'] >>> import keyword as _keyword >>> get_close_matches("wheel", _keyword.kwlist) ['while'] >>> get_close_matches("apple", _keyword.kwlist) [] >>> get_close_matches("accept", _keyword.kwlist) ['except'] """ if not n > 0: raise ValueError("n must be > 0: %r" % (n,)) if not 0.0 <= cutoff <= 1.0: raise ValueError("cutoff must be in [0.0, 1.0]: %r" % (cutoff,)) result = [] s = SequenceMatcher() s.set_seq2(word) for x in possibilities: s.set_seq1(x) if s.real_quick_ratio() >= cutoff and \ s.quick_ratio() >= cutoff and \ s.ratio() >= cutoff: result.append((s.ratio(), x)) # Move the best scorers to head of list result = heapq.nlargest(n, result) # Strip scores for the best n matches return [x for score, x in result] def _count_leading(line, ch): """ Return number of `ch` characters at the start of `line`. Example: >>> _count_leading(' abc', ' ') 3 """ i, n = 0, len(line) while i < n and line[i] == ch: i += 1 return i class Differ(object): r""" Differ is a class for comparing sequences of lines of text, and producing human-readable differences or deltas. Differ uses SequenceMatcher both to compare sequences of lines, and to compare sequences of characters within similar (near-matching) lines. Each line of a Differ delta begins with a two-letter code: '- ' line unique to sequence 1 '+ ' line unique to sequence 2 ' ' line common to both sequences '? ' line not present in either input sequence Lines beginning with '? ' attempt to guide the eye to intraline differences, and were not present in either input sequence. These lines can be confusing if the sequences contain tab characters. Note that Differ makes no claim to produce a *minimal* diff. To the contrary, minimal diffs are often counter-intuitive, because they synch up anywhere possible, sometimes accidental matches 100 pages apart. Restricting synch points to contiguous matches preserves some notion of locality, at the occasional cost of producing a longer diff. Example: Comparing two texts. First we set up the texts, sequences of individual single-line strings ending with newlines (such sequences can also be obtained from the `readlines()` method of file-like objects): >>> text1 = ''' 1. Beautiful is better than ugly. ... 2. Explicit is better than implicit. ... 3. Simple is better than complex. ... 4. Complex is better than complicated. ... '''.splitlines(1) >>> len(text1) 4 >>> text1[0][-1] '\n' >>> text2 = ''' 1. Beautiful is better than ugly. ... 3. Simple is better than complex. ... 4. Complicated is better than complex. ... 5. Flat is better than nested. ... '''.splitlines(1) Next we instantiate a Differ object: >>> d = Differ() Note that when instantiating a Differ object we may pass functions to filter out line and character 'junk'. See Differ.__init__ for details. Finally, we compare the two: >>> result = list(d.compare(text1, text2)) 'result' is a list of strings, so let's pretty-print it: >>> from pprint import pprint as _pprint >>> _pprint(result) [' 1. Beautiful is better than ugly.\n', '- 2. Explicit is better than implicit.\n', '- 3. Simple is better than complex.\n', '+ 3. Simple is better than complex.\n', '? ++\n', '- 4. Complex is better than complicated.\n', '? ^ ---- ^\n', '+ 4. Complicated is better than complex.\n', '? ++++ ^ ^\n', '+ 5. Flat is better than nested.\n'] As a single multi-line string it looks like this: >>> print ''.join(result), 1. Beautiful is better than ugly. - 2. Explicit is better than implicit. - 3. Simple is better than complex. + 3. Simple is better than complex. ? ++ - 4. Complex is better than complicated. ? ^ ---- ^ + 4. Complicated is better than complex. ? ++++ ^ ^ + 5. Flat is better than nested. Methods: __init__(linejunk=None, charjunk=None) Construct a text differencer, with optional filters. compare(a, b) Compare two sequences of lines; generate the resulting delta. """ def __init__(self, linejunk=None, charjunk=None): """ Construct a text differencer, with optional filters. The two optional keyword parameters are for filter functions: - `linejunk`: A function that should accept a single string argument, and return true iff the string is junk. The module-level function `IS_LINE_JUNK` may be used to filter out lines without visible characters, except for at most one splat ('#'). It is recommended to leave linejunk None; as of Python 2.3, the underlying SequenceMatcher class has grown an adaptive notion of "noise" lines that's better than any static definition the author has ever been able to craft. - `charjunk`: A function that should accept a string of length 1. The module-level function `IS_CHARACTER_JUNK` may be used to filter out whitespace characters (a blank or tab; **note**: bad idea to include newline in this!). Use of IS_CHARACTER_JUNK is recommended. """ self.linejunk = linejunk self.charjunk = charjunk def compare(self, a, b): r""" Compare two sequences of lines; generate the resulting delta. Each sequence must contain individual single-line strings ending with newlines. Such sequences can be obtained from the `readlines()` method of file-like objects. The delta generated also consists of newline- terminated strings, ready to be printed as-is via the writeline() method of a file-like object. Example: >>> print ''.join(Differ().compare('one\ntwo\nthree\n'.splitlines(1), ... 'ore\ntree\nemu\n'.splitlines(1))), - one ? ^ + ore ? ^ - two - three ? - + tree + emu """ cruncher = SequenceMatcher(self.linejunk, a, b) for tag, alo, ahi, blo, bhi in cruncher.get_opcodes(): if tag == 'replace': g = self._fancy_replace(a, alo, ahi, b, blo, bhi) elif tag == 'delete': g = self._dump('-', a, alo, ahi) elif tag == 'insert': g = self._dump('+', b, blo, bhi) elif tag == 'equal': g = self._dump(' ', a, alo, ahi) else: raise ValueError, 'unknown tag %r' % (tag,) for line in g: yield line def _dump(self, tag, x, lo, hi): """Generate comparison results for a same-tagged range.""" for i in xrange(lo, hi): yield '%s %s' % (tag, x[i]) def _plain_replace(self, a, alo, ahi, b, blo, bhi): assert alo < ahi and blo < bhi # dump the shorter block first -- reduces the burden on short-term # memory if the blocks are of very different sizes if bhi - blo < ahi - alo: first = self._dump('+', b, blo, bhi) second = self._dump('-', a, alo, ahi) else: first = self._dump('-', a, alo, ahi) second = self._dump('+', b, blo, bhi) for g in first, second: for line in g: yield line def _fancy_replace(self, a, alo, ahi, b, blo, bhi): r""" When replacing one block of lines with another, search the blocks for *similar* lines; the best-matching pair (if any) is used as a synch point, and intraline difference marking is done on the similar pair. Lots of work, but often worth it. Example: >>> d = Differ() >>> results = d._fancy_replace(['abcDefghiJkl\n'], 0, 1, ... ['abcdefGhijkl\n'], 0, 1) >>> print ''.join(results), - abcDefghiJkl ? ^ ^ ^ + abcdefGhijkl ? ^ ^ ^ """ # don't synch up unless the lines have a similarity score of at # least cutoff; best_ratio tracks the best score seen so far best_ratio, cutoff = 0.74, 0.75 cruncher = SequenceMatcher(self.charjunk) eqi, eqj = None, None # 1st indices of equal lines (if any) # search for the pair that matches best without being identical # (identical lines must be junk lines, & we don't want to synch up # on junk -- unless we have to) for j in xrange(blo, bhi): bj = b[j] cruncher.set_seq2(bj) for i in xrange(alo, ahi): ai = a[i] if ai == bj: if eqi is None: eqi, eqj = i, j continue cruncher.set_seq1(ai) # computing similarity is expensive, so use the quick # upper bounds first -- have seen this speed up messy # compares by a factor of 3. # note that ratio() is only expensive to compute the first # time it's called on a sequence pair; the expensive part # of the computation is cached by cruncher if cruncher.real_quick_ratio() > best_ratio and \ cruncher.quick_ratio() > best_ratio and \ cruncher.ratio() > best_ratio: best_ratio, best_i, best_j = cruncher.ratio(), i, j if best_ratio < cutoff: # no non-identical "pretty close" pair if eqi is None: # no identical pair either -- treat it as a straight replace for line in self._plain_replace(a, alo, ahi, b, blo, bhi): yield line return # no close pair, but an identical pair -- synch up on that best_i, best_j, best_ratio = eqi, eqj, 1.0 else: # there's a close pair, so forget the identical pair (if any) eqi = None # a[best_i] very similar to b[best_j]; eqi is None iff they're not # identical # pump out diffs from before the synch point for line in self._fancy_helper(a, alo, best_i, b, blo, best_j): yield line # do intraline marking on the synch pair aelt, belt = a[best_i], b[best_j] if eqi is None: # pump out a '-', '?', '+', '?' quad for the synched lines atags = btags = "" cruncher.set_seqs(aelt, belt) for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): la, lb = ai2 - ai1, bj2 - bj1 if tag == 'replace': atags += '^' * la btags += '^' * lb elif tag == 'delete': atags += '-' * la elif tag == 'insert': btags += '+' * lb elif tag == 'equal': atags += ' ' * la btags += ' ' * lb else: raise ValueError, 'unknown tag %r' % (tag,) for line in self._qformat(aelt, belt, atags, btags): yield line else: # the synch pair is identical yield ' ' + aelt # pump out diffs from after the synch point for line in self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi): yield line def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: g = self._fancy_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: g = self._dump('+', b, blo, bhi) for line in g: yield line def _qformat(self, aline, bline, atags, btags): r""" Format "?" output and deal with leading tabs. Example: >>> d = Differ() >>> results = d._qformat('\tabcDefghiJkl\n', '\tabcdefGhijkl\n', ... ' ^ ^ ^ ', ' ^ ^ ^ ') >>> for line in results: print repr(line) ... '- \tabcDefghiJkl\n' '? \t ^ ^ ^\n' '+ \tabcdefGhijkl\n' '? \t ^ ^ ^\n' """ # Can hurt, but will probably help most of the time. common = min(_count_leading(aline, "\t"), _count_leading(bline, "\t")) common = min(common, _count_leading(atags[:common], " ")) common = min(common, _count_leading(btags[:common], " ")) atags = atags[common:].rstrip() btags = btags[common:].rstrip() yield "- " + aline if atags: yield "? %s%s\n" % ("\t" * common, atags) yield "+ " + bline if btags: yield "? %s%s\n" % ("\t" * common, btags) # With respect to junk, an earlier version of ndiff simply refused to # *start* a match with a junk element. The result was cases like this: # before: private Thread currentThread; # after: private volatile Thread currentThread; # If you consider whitespace to be junk, the longest contiguous match # not starting with junk is "e Thread currentThread". So ndiff reported # that "e volatil" was inserted between the 't' and the 'e' in "private". # While an accurate view, to people that's absurd. The current version # looks for matching blocks that are entirely junk-free, then extends the # longest one of those as far as possible but only with matching junk. # So now "currentThread" is matched, then extended to suck up the # preceding blank; then "private" is matched, and extended to suck up the # following blank; then "Thread" is matched; and finally ndiff reports # that "volatile " was inserted before "Thread". The only quibble # remaining is that perhaps it was really the case that " volatile" # was inserted after "private". I can live with that . import re def IS_LINE_JUNK(line, pat=re.compile(r"\s*#?\s*$").match): r""" Return 1 for ignorable line: iff `line` is blank or contains a single '#'. Examples: >>> IS_LINE_JUNK('\n') True >>> IS_LINE_JUNK(' # \n') True >>> IS_LINE_JUNK('hello\n') False """ return pat(line) is not None def IS_CHARACTER_JUNK(ch, ws=" \t"): r""" Return 1 for ignorable character: iff `ch` is a space or tab. Examples: >>> IS_CHARACTER_JUNK(' ') True >>> IS_CHARACTER_JUNK('\t') True >>> IS_CHARACTER_JUNK('\n') False >>> IS_CHARACTER_JUNK('x') False """ return ch in ws ######################################################################## ### Unified Diff ######################################################################## def _format_range_unified(start, stop): 'Convert range to the "ed" format' # Per the diff spec at http://www.unix.org/single_unix_specification/ beginning = start + 1 # lines start numbering with one length = stop - start if length == 1: # return '{}'.format(beginning) return '%s' % (beginning) if not length: beginning -= 1 # empty ranges begin at line just before the range return '%s,%s' % (beginning, length) def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): r""" Compare two sequences of lines; generate the delta as a unified diff. Unified diffs are a compact way of showing line changes and a few lines of context. The number of context lines is set by 'n' which defaults to three. By default, the diff control lines (those with ---, +++, or @@) are created with a trailing newline. This is helpful so that inputs created from file.readlines() result in diffs that are suitable for file.writelines() since both the inputs and outputs have trailing newlines. For inputs that do not have trailing newlines, set the lineterm argument to "" so that the output will be uniformly newline free. The unidiff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. The modification times are normally expressed in the ISO 8601 format. Example: >>> for line in unified_diff('one two three four'.split(), ... 'zero one tree four'.split(), 'Original', 'Current', ... '2005-01-26 23:30:50', '2010-04-02 10:20:52', ... lineterm=''): ... print line # doctest: +NORMALIZE_WHITESPACE --- Original 2005-01-26 23:30:50 +++ Current 2010-04-02 10:20:52 @@ -1,4 +1,4 @@ +zero one -two -three +tree four """ started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: started = True # fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' fromdate = '\t%s' % (fromfiledate) if fromfiledate else '' # todate = '\t{}'.format(tofiledate) if tofiledate else '' todate = '\t%s' % (tofiledate) if tofiledate else '' # yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) yield '--- %s%s%s' % (fromfile, fromdate, lineterm) # yield '+++ {}{}{}'.format(tofile, todate, lineterm) yield '+++ %s%s%s' % (tofile, todate, lineterm) first, last = group[0], group[-1] file1_range = _format_range_unified(first[1], last[2]) file2_range = _format_range_unified(first[3], last[4]) # yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) yield '@@ -%s +%s @@%s' % (file1_range, file2_range, lineterm) for tag, i1, i2, j1, j2 in group: if tag == 'equal': for line in a[i1:i2]: yield ' ' + line continue if tag in ('replace', 'delete'): for line in a[i1:i2]: yield '-' + line if tag in ('replace', 'insert'): for line in b[j1:j2]: yield '+' + line ######################################################################## ### Context Diff ######################################################################## def _format_range_context(start, stop): 'Convert range to the "ed" format' # Per the diff spec at http://www.unix.org/single_unix_specification/ beginning = start + 1 # lines start numbering with one length = stop - start if not length: beginning -= 1 # empty ranges begin at line just before the range if length <= 1: # return '{}'.format(beginning) return '%s' % (beginning) # return '{},{}'.format(beginning, beginning + length - 1) return '%s,%s' % (beginning, beginning + length - 1) # See http://www.unix.org/single_unix_specification/ def context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): r""" Compare two sequences of lines; generate the delta as a context diff. Context diffs are a compact way of showing line changes and a few lines of context. The number of context lines is set by 'n' which defaults to three. By default, the diff control lines (those with *** or ---) are created with a trailing newline. This is helpful so that inputs created from file.readlines() result in diffs that are suitable for file.writelines() since both the inputs and outputs have trailing newlines. For inputs that do not have trailing newlines, set the lineterm argument to "" so that the output will be uniformly newline free. The context diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. The modification times are normally expressed in the ISO 8601 format. If not specified, the strings default to blanks. Example: >>> print ''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(1), ... 'zero\none\ntree\nfour\n'.splitlines(1), 'Original', 'Current')), *** Original --- Current *************** *** 1,4 **** one ! two ! three four --- 1,4 ---- + zero one ! tree four """ prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ') started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: started = True # fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' fromdate = '\t%s' % (fromfiledate) if fromfiledate else '' # todate = '\t{}'.format(tofiledate) if tofiledate else '' todate = '\t%s' % (tofiledate) if tofiledate else '' # yield '*** {}{}{}'.format(fromfile, fromdate, lineterm) yield '*** %s%s%s' % (fromfile, fromdate, lineterm) # yield '--- {}{}{}'.format(tofile, todate, lineterm) yield '--- %s%s%s' % (tofile, todate, lineterm) first, last = group[0], group[-1] yield '***************' + lineterm file1_range = _format_range_context(first[1], last[2]) # yield '*** {} ****{}'.format(file1_range, lineterm) yield '*** %s ****%s' % (file1_range, lineterm) if any(tag in ('replace', 'delete') for tag, _, _, _, _ in group): for tag, i1, i2, _, _ in group: if tag != 'insert': for line in a[i1:i2]: yield prefix[tag] + line file2_range = _format_range_context(first[3], last[4]) # yield '--- {} ----{}'.format(file2_range, lineterm) yield '--- %s ----%s' % (file2_range, lineterm) if any(tag in ('replace', 'insert') for tag, _, _, _, _ in group): for tag, _, _, j1, j2 in group: if tag != 'delete': for line in b[j1:j2]: yield prefix[tag] + line def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): r""" Compare `a` and `b` (lists of strings); return a `Differ`-style delta. Optional keyword parameters `linejunk` and `charjunk` are for filter functions (or None): - linejunk: A function that should accept a single string argument, and return true iff the string is junk. The default is None, and is recommended; as of Python 2.3, an adaptive notion of "noise" lines is used that does a good job on its own. - charjunk: A function that should accept a string of length 1. The default is module-level function IS_CHARACTER_JUNK, which filters out whitespace characters (a blank or tab; note: bad idea to include newline in this!). Tools/scripts/ndiff.py is a command-line front-end to this function. Example: >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1), ... 'ore\ntree\nemu\n'.splitlines(1)) >>> print ''.join(diff), - one ? ^ + ore ? ^ - two - three ? - + tree + emu """ return Differ(linejunk, charjunk).compare(a, b) def _mdiff(fromlines, tolines, context=None, linejunk=None, charjunk=IS_CHARACTER_JUNK): r"""Returns generator yielding marked up from/to side by side differences. Arguments: fromlines -- list of text lines to compared to tolines tolines -- list of text lines to be compared to fromlines context -- number of context lines to display on each side of difference, if None, all from/to text lines will be generated. linejunk -- passed on to ndiff (see ndiff documentation) charjunk -- passed on to ndiff (see ndiff documentation) This function returns an iterator which returns a tuple: (from line tuple, to line tuple, boolean flag) from/to line tuple -- (line num, line text) line num -- integer or None (to indicate a context separation) line text -- original line text with following markers inserted: '\0+' -- marks start of added text '\0-' -- marks start of deleted text '\0^' -- marks start of changed text '\1' -- marks end of added/deleted/changed text boolean flag -- None indicates context separation, True indicates either "from" or "to" line contains a change, otherwise False. This function/iterator was originally developed to generate side by side file difference for making HTML pages (see HtmlDiff class for example usage). Note, this function utilizes the ndiff function to generate the side by side difference markup. Optional ndiff arguments may be passed to this function and they in turn will be passed to ndiff. """ import re # regular expression for finding intraline change indices change_re = re.compile('(\++|\-+|\^+)') # create the difference iterator to generate the differences diff_lines_iterator = ndiff(fromlines,tolines,linejunk,charjunk) def _make_line(lines, format_key, side, num_lines=[0,0]): """Returns line of text with user's change markup and line formatting. lines -- list of lines from the ndiff generator to produce a line of text from. When producing the line of text to return, the lines used are removed from this list. format_key -- '+' return first line in list with "add" markup around the entire line. '-' return first line in list with "delete" markup around the entire line. '?' return first line in list with add/delete/change intraline markup (indices obtained from second line) None return first line in list with no markup side -- indice into the num_lines list (0=from,1=to) num_lines -- from/to current line number. This is NOT intended to be a passed parameter. It is present as a keyword argument to maintain memory of the current line numbers between calls of this function. Note, this function is purposefully not defined at the module scope so that data it needs from its parent function (within whose context it is defined) does not need to be of module scope. """ num_lines[side] += 1 # Handle case where no user markup is to be added, just return line of # text with user's line format to allow for usage of the line number. if format_key is None: return (num_lines[side],lines.pop(0)[2:]) # Handle case of intraline changes if format_key == '?': text, markers = lines.pop(0), lines.pop(0) # find intraline changes (store change type and indices in tuples) sub_info = [] def record_sub_info(match_object,sub_info=sub_info): sub_info.append([match_object.group(1)[0],match_object.span()]) return match_object.group(1) change_re.sub(record_sub_info,markers) # process each tuple inserting our special marks that won't be # noticed by an xml/html escaper. for key,(begin,end) in sub_info[::-1]: text = text[0:begin]+'\0'+key+text[begin:end]+'\1'+text[end:] text = text[2:] # Handle case of add/delete entire line else: text = lines.pop(0)[2:] # if line of text is just a newline, insert a space so there is # something for the user to highlight and see. if not text: text = ' ' # insert marks that won't be noticed by an xml/html escaper. text = '\0' + format_key + text + '\1' # Return line of text, first allow user's line formatter to do its # thing (such as adding the line number) then replace the special # marks with what the user's change markup. return (num_lines[side],text) def _line_iterator(): """Yields from/to lines of text with a change indication. This function is an iterator. It itself pulls lines from a differencing iterator, processes them and yields them. When it can it yields both a "from" and a "to" line, otherwise it will yield one or the other. In addition to yielding the lines of from/to text, a boolean flag is yielded to indicate if the text line(s) have differences in them. Note, this function is purposefully not defined at the module scope so that data it needs from its parent function (within whose context it is defined) does not need to be of module scope. """ lines = [] num_blanks_pending, num_blanks_to_yield = 0, 0 while True: # Load up next 4 lines so we can look ahead, create strings which # are a concatenation of the first character of each of the 4 lines # so we can do some very readable comparisons. while len(lines) < 4: try: lines.append(diff_lines_iterator.next()) except StopIteration: lines.append('X') s = ''.join([line[0] for line in lines]) if s.startswith('X'): # When no more lines, pump out any remaining blank lines so the # corresponding add/delete lines get a matching blank line so # all line pairs get yielded at the next level. num_blanks_to_yield = num_blanks_pending elif s.startswith('-?+?'): # simple intraline change yield _make_line(lines,'?',0), _make_line(lines,'?',1), True continue elif s.startswith('--++'): # in delete block, add block coming: we do NOT want to get # caught up on blank lines yet, just process the delete line num_blanks_pending -= 1 yield _make_line(lines,'-',0), None, True continue elif s.startswith(('--?+', '--+', '- ')): # in delete block and see an intraline change or unchanged line # coming: yield the delete line and then blanks from_line,to_line = _make_line(lines,'-',0), None num_blanks_to_yield,num_blanks_pending = num_blanks_pending-1,0 elif s.startswith('-+?'): # intraline change yield _make_line(lines,None,0), _make_line(lines,'?',1), True continue elif s.startswith('-?+'): # intraline change yield _make_line(lines,'?',0), _make_line(lines,None,1), True continue elif s.startswith('-'): # delete FROM line num_blanks_pending -= 1 yield _make_line(lines,'-',0), None, True continue elif s.startswith('+--'): # in add block, delete block coming: we do NOT want to get # caught up on blank lines yet, just process the add line num_blanks_pending += 1 yield None, _make_line(lines,'+',1), True continue elif s.startswith(('+ ', '+-')): # will be leaving an add block: yield blanks then add line from_line, to_line = None, _make_line(lines,'+',1) num_blanks_to_yield,num_blanks_pending = num_blanks_pending+1,0 elif s.startswith('+'): # inside an add block, yield the add line num_blanks_pending += 1 yield None, _make_line(lines,'+',1), True continue elif s.startswith(' '): # unchanged text, yield it to both sides yield _make_line(lines[:],None,0),_make_line(lines,None,1),False continue # Catch up on the blank lines so when we yield the next from/to # pair, they are lined up. while(num_blanks_to_yield < 0): num_blanks_to_yield += 1 yield None,('','\n'),True while(num_blanks_to_yield > 0): num_blanks_to_yield -= 1 yield ('','\n'),None,True if s.startswith('X'): raise StopIteration else: yield from_line,to_line,True def _line_pair_iterator(): """Yields from/to lines of text with a change indication. This function is an iterator. It itself pulls lines from the line iterator. Its difference from that iterator is that this function always yields a pair of from/to text lines (with the change indication). If necessary it will collect single from/to lines until it has a matching pair from/to pair to yield. Note, this function is purposefully not defined at the module scope so that data it needs from its parent function (within whose context it is defined) does not need to be of module scope. """ line_iterator = _line_iterator() fromlines,tolines=[],[] while True: # Collecting lines of text until we have a from/to pair while (len(fromlines)==0 or len(tolines)==0): from_line, to_line, found_diff =line_iterator.next() if from_line is not None: fromlines.append((from_line,found_diff)) if to_line is not None: tolines.append((to_line,found_diff)) # Once we have a pair, remove them from the collection and yield it from_line, fromDiff = fromlines.pop(0) to_line, to_diff = tolines.pop(0) yield (from_line,to_line,fromDiff or to_diff) # Handle case where user does not want context differencing, just yield # them up without doing anything else with them. line_pair_iterator = _line_pair_iterator() if context is None: while True: yield line_pair_iterator.next() # Handle case where user wants context differencing. We must do some # storage of lines until we know for sure that they are to be yielded. else: context += 1 lines_to_write = 0 while True: # Store lines up until we find a difference, note use of a # circular queue because we only need to keep around what # we need for context. index, contextLines = 0, [None]*(context) found_diff = False while(found_diff is False): from_line, to_line, found_diff = line_pair_iterator.next() i = index % context contextLines[i] = (from_line, to_line, found_diff) index += 1 # Yield lines that we have collected so far, but first yield # the user's separator. if index > context: yield None, None, None lines_to_write = context else: lines_to_write = index index = 0 while(lines_to_write): i = index % context index += 1 yield contextLines[i] lines_to_write -= 1 # Now yield the context lines after the change lines_to_write = context-1 while(lines_to_write): from_line, to_line, found_diff = line_pair_iterator.next() # If another change within the context, extend the context if found_diff: lines_to_write = context-1 else: lines_to_write -= 1 yield from_line, to_line, found_diff _file_template = """ %(table)s%(legend)s """ _styles = """ table.diff {font-family:Courier; border:medium;} .diff_header {background-color:#e0e0e0} td.diff_header {text-align:right} .diff_next {background-color:#c0c0c0} .diff_add {background-color:#aaffaa} .diff_chg {background-color:#ffff77} .diff_sub {background-color:#ffaaaa}""" _table_template = """ %(header_row)s %(data_rows)s
""" _legend = """
Legends
Colors
 Added 
Changed
Deleted
Links
(f)irst change
(n)ext change
(t)op
""" class HtmlDiff(object): """For producing HTML side by side comparison with change highlights. This class can be used to create an HTML table (or a complete HTML file containing the table) showing a side by side, line by line comparison of text with inter-line and intra-line change highlights. The table can be generated in either full or contextual difference mode. The following methods are provided for HTML generation: make_table -- generates HTML for a single side by side table make_file -- generates complete HTML file with a single side by side table See tools/scripts/diff.py for an example usage of this class. """ _file_template = _file_template _styles = _styles _table_template = _table_template _legend = _legend _default_prefix = 0 def __init__(self,tabsize=8,wrapcolumn=None,linejunk=None, charjunk=IS_CHARACTER_JUNK): """HtmlDiff instance initializer Arguments: tabsize -- tab stop spacing, defaults to 8. wrapcolumn -- column number where lines are broken and wrapped, defaults to None where lines are not wrapped. linejunk,charjunk -- keyword arguments passed into ndiff() (used to by HtmlDiff() to generate the side by side HTML differences). See ndiff() documentation for argument default values and descriptions. """ self._tabsize = tabsize self._wrapcolumn = wrapcolumn self._linejunk = linejunk self._charjunk = charjunk def make_file(self,fromlines,tolines,fromdesc='',todesc='',context=False, numlines=5): """Returns HTML file of side by side comparison with change highlights Arguments: fromlines -- list of "from" lines tolines -- list of "to" lines fromdesc -- "from" file column header string todesc -- "to" file column header string context -- set to True for contextual differences (defaults to False which shows full differences). numlines -- number of context lines. When context is set True, controls number of lines displayed before and after the change. When context is False, controls the number of lines to place the "next" link anchors before the next change (so click of "next" link jumps to just before the change). """ return self._file_template % dict( styles = self._styles, legend = self._legend, table = self.make_table(fromlines,tolines,fromdesc,todesc, context=context,numlines=numlines)) def _tab_newline_replace(self,fromlines,tolines): """Returns from/to line lists with tabs expanded and newlines removed. Instead of tab characters being replaced by the number of spaces needed to fill in to the next tab stop, this function will fill the space with tab characters. This is done so that the difference algorithms can identify changes in a file when tabs are replaced by spaces and vice versa. At the end of the HTML generation, the tab characters will be replaced with a nonbreakable space. """ def expand_tabs(line): # hide real spaces line = line.replace(' ','\0') # expand tabs into spaces line = line.expandtabs(self._tabsize) # replace spaces from expanded tabs back into tab characters # (we'll replace them with markup after we do differencing) line = line.replace(' ','\t') return line.replace('\0',' ').rstrip('\n') fromlines = [expand_tabs(line) for line in fromlines] tolines = [expand_tabs(line) for line in tolines] return fromlines,tolines def _split_line(self,data_list,line_num,text): """Builds list of text lines by splitting text lines at wrap point This function will determine if the input text line needs to be wrapped (split) into separate lines. If so, the first wrap point will be determined and the first line appended to the output text line list. This function is used recursively to handle the second part of the split line to further split it. """ # if blank line or context separator, just add it to the output list if not line_num: data_list.append((line_num,text)) return # if line text doesn't need wrapping, just add it to the output list size = len(text) max = self._wrapcolumn if (size <= max) or ((size -(text.count('\0')*3)) <= max): data_list.append((line_num,text)) return # scan text looking for the wrap point, keeping track if the wrap # point is inside markers i = 0 n = 0 mark = '' while n < max and i < size: if text[i] == '\0': i += 1 mark = text[i] i += 1 elif text[i] == '\1': i += 1 mark = '' else: i += 1 n += 1 # wrap point is inside text, break it up into separate lines line1 = text[:i] line2 = text[i:] # if wrap point is inside markers, place end marker at end of first # line and start marker at beginning of second line because each # line will have its own table tag markup around it. if mark: line1 = line1 + '\1' line2 = '\0' + mark + line2 # tack on first line onto the output list data_list.append((line_num,line1)) # use this routine again to wrap the remaining text self._split_line(data_list,'>',line2) def _line_wrapper(self,diffs): """Returns iterator that splits (wraps) mdiff text lines""" # pull from/to data and flags from mdiff iterator for fromdata,todata,flag in diffs: # check for context separators and pass them through if flag is None: yield fromdata,todata,flag continue (fromline,fromtext),(toline,totext) = fromdata,todata # for each from/to line split it at the wrap column to form # list of text lines. fromlist,tolist = [],[] self._split_line(fromlist,fromline,fromtext) self._split_line(tolist,toline,totext) # yield from/to line in pairs inserting blank lines as # necessary when one side has more wrapped lines while fromlist or tolist: if fromlist: fromdata = fromlist.pop(0) else: fromdata = ('',' ') if tolist: todata = tolist.pop(0) else: todata = ('',' ') yield fromdata,todata,flag def _collect_lines(self,diffs): """Collects mdiff output into separate lists Before storing the mdiff from/to data into a list, it is converted into a single line of text with HTML markup. """ fromlist,tolist,flaglist = [],[],[] # pull from/to data and flags from mdiff style iterator for fromdata,todata,flag in diffs: try: # store HTML markup of the lines into the lists fromlist.append(self._format_line(0,flag,*fromdata)) tolist.append(self._format_line(1,flag,*todata)) except TypeError: # exceptions occur for lines where context separators go fromlist.append(None) tolist.append(None) flaglist.append(flag) return fromlist,tolist,flaglist def _format_line(self,side,flag,linenum,text): """Returns HTML markup of "from" / "to" text lines side -- 0 or 1 indicating "from" or "to" text flag -- indicates if difference on line linenum -- line number (used for line number column) text -- line text to be marked up """ try: linenum = '%d' % linenum id = ' id="%s%s"' % (self._prefix[side],linenum) except TypeError: # handle blank lines where linenum is '>' or '' id = '' # replace those things that would get confused with HTML symbols text=text.replace("&","&").replace(">",">").replace("<","<") # make space non-breakable so they don't get compressed or line wrapped text = text.replace(' ',' ').rstrip() return '%s%s' \ % (id,linenum,text) def _make_prefix(self): """Create unique anchor prefixes""" # Generate a unique anchor prefix so multiple tables # can exist on the same HTML page without conflicts. fromprefix = "from%d_" % HtmlDiff._default_prefix toprefix = "to%d_" % HtmlDiff._default_prefix HtmlDiff._default_prefix += 1 # store prefixes so line format method has access self._prefix = [fromprefix,toprefix] def _convert_flags(self,fromlist,tolist,flaglist,context,numlines): """Makes list of "next" links""" # all anchor names will be generated using the unique "to" prefix toprefix = self._prefix[1] # process change flags, generating middle column of next anchors/links next_id = ['']*len(flaglist) next_href = ['']*len(flaglist) num_chg, in_change = 0, False last = 0 for i,flag in enumerate(flaglist): if flag: if not in_change: in_change = True last = i # at the beginning of a change, drop an anchor a few lines # (the context lines) before the change for the previous # link i = max([0,i-numlines]) next_id[i] = ' id="difflib_chg_%s_%d"' % (toprefix,num_chg) # at the beginning of a change, drop a link to the next # change num_chg += 1 next_href[last] = 'n' % ( toprefix,num_chg) else: in_change = False # check for cases where there is no content to avoid exceptions if not flaglist: flaglist = [False] next_id = [''] next_href = [''] last = 0 if context: fromlist = [' No Differences Found '] tolist = fromlist else: fromlist = tolist = [' Empty File '] # if not a change on first line, drop a link if not flaglist[0]: next_href[0] = 'f' % toprefix # redo the last link to link to the top next_href[last] = 't' % (toprefix) return fromlist,tolist,flaglist,next_href,next_id def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False, numlines=5): """Returns HTML table of side by side comparison with change highlights Arguments: fromlines -- list of "from" lines tolines -- list of "to" lines fromdesc -- "from" file column header string todesc -- "to" file column header string context -- set to True for contextual differences (defaults to False which shows full differences). numlines -- number of context lines. When context is set True, controls number of lines displayed before and after the change. When context is False, controls the number of lines to place the "next" link anchors before the next change (so click of "next" link jumps to just before the change). """ # make unique anchor prefixes so that multiple tables may exist # on the same page without conflict. self._make_prefix() # change tabs to spaces before it gets more difficult after we insert # markup fromlines,tolines = self._tab_newline_replace(fromlines,tolines) # create diffs iterator which generates side by side from/to data if context: context_lines = numlines else: context_lines = None diffs = _mdiff(fromlines,tolines,context_lines,linejunk=self._linejunk, charjunk=self._charjunk) # set up iterator to wrap lines that exceed desired width if self._wrapcolumn: diffs = self._line_wrapper(diffs) # collect up from/to lines and flags into lists (also format the lines) fromlist,tolist,flaglist = self._collect_lines(diffs) # process change flags, generating middle column of next anchors/links fromlist,tolist,flaglist,next_href,next_id = self._convert_flags( fromlist,tolist,flaglist,context,numlines) s = [] fmt = ' %s%s' + \ '%s%s\n' for i in range(len(flaglist)): if flaglist[i] is None: # mdiff yields None on separator lines skip the bogus ones # generated for the first line if i > 0: s.append(' \n \n') else: s.append( fmt % (next_id[i],next_href[i],fromlist[i], next_href[i],tolist[i])) if fromdesc or todesc: header_row = '%s%s%s%s' % ( '
', '%s' % fromdesc, '
', '%s' % todesc) else: header_row = '' table = self._table_template % dict( data_rows=''.join(s), header_row=header_row, prefix=self._prefix[1]) return table.replace('\0+',''). \ replace('\0-',''). \ replace('\0^',''). \ replace('\1',''). \ replace('\t',' ') del re def restore(delta, which): r""" Generate one of the two sequences that generated a delta. Given a `delta` produced by `Differ.compare()` or `ndiff()`, extract lines originating from file 1 or 2 (parameter `which`), stripping off line prefixes. Examples: >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1), ... 'ore\ntree\nemu\n'.splitlines(1)) >>> diff = list(diff) >>> print ''.join(restore(diff, 1)), one two three >>> print ''.join(restore(diff, 2)), ore tree emu """ try: tag = {1: "- ", 2: "+ "}[int(which)] except KeyError: raise ValueError, ('unknown delta choice (must be 1 or 2): %r' % which) prefixes = (" ", tag) for line in delta: if line[:2] in prefixes: yield line[2:] # def _test(): # import doctest, difflib # return doctest.testmod(difflib) # if __name__ == "__main__": # _test() ================================================ FILE: third_party/stdlib/dircache.py ================================================ """Read and cache directory listings. The listdir() routine returns a sorted list of the files in a directory, using a cache to avoid reading the directory more often than necessary. The annotate() routine appends slashes to directories.""" from warnings import warnpy3k warnpy3k("the dircache module has been removed in Python 3.0", stacklevel=2) del warnpy3k import os __all__ = ["listdir", "opendir", "annotate", "reset"] cache = {} def reset(): """Reset the cache completely.""" global cache cache = {} def listdir(path): """List directory contents, using cache.""" try: cached_mtime, list = cache[path] del cache[path] except KeyError: cached_mtime, list = -1, [] mtime = os.stat(path).st_mtime if mtime != cached_mtime: list = os.listdir(path) list.sort() cache[path] = mtime, list return list opendir = listdir # XXX backward compatibility def annotate(head, list): """Add '/' suffixes to directories.""" for i in range(len(list)): if os.path.isdir(os.path.join(head, list[i])): list[i] = list[i] + '/' ================================================ FILE: third_party/stdlib/dummy_thread.py ================================================ """Drop-in replacement for the thread module. Meant to be used as a brain-dead substitute so that threaded code does not need to be rewritten for when the thread module is not present. Suggested usage is:: try: import thread except ImportError: import dummy_thread as thread """ # Exports only things specified by thread documentation; # skipping obsolete synonyms allocate(), start_new(), exit_thread(). __all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock', 'interrupt_main', 'LockType'] import traceback as _traceback class error(Exception): """Dummy implementation of thread.error.""" def __init__(self, *args): self.args = args def start_new_thread(function, args, kwargs={}): """Dummy implementation of thread.start_new_thread(). Compatibility is maintained by making sure that ``args`` is a tuple and ``kwargs`` is a dictionary. If an exception is raised and it is SystemExit (which can be done by thread.exit()) it is caught and nothing is done; all other exceptions are printed out by using traceback.print_exc(). If the executed function calls interrupt_main the KeyboardInterrupt will be raised when the function returns. """ if type(args) != type(tuple()): raise TypeError("2nd arg must be a tuple") if type(kwargs) != type(dict()): raise TypeError("3rd arg must be a dict") global _main _main = False try: function(*args, **kwargs) except SystemExit: pass except: _traceback.print_exc() _main = True global _interrupt if _interrupt: _interrupt = False raise KeyboardInterrupt def exit(): """Dummy implementation of thread.exit().""" raise SystemExit def get_ident(): """Dummy implementation of thread.get_ident(). Since this module should only be used when threadmodule is not available, it is safe to assume that the current process is the only thread. Thus a constant can be safely returned. """ return -1 def allocate_lock(): """Dummy implementation of thread.allocate_lock().""" return LockType() def stack_size(size=None): """Dummy implementation of thread.stack_size().""" if size is not None: raise error("setting thread stack size not supported") return 0 class LockType(object): """Class implementing dummy implementation of thread.LockType. Compatibility is maintained by maintaining self.locked_status which is a boolean that stores the state of the lock. Pickling of the lock, though, should not be done since if the thread module is then used with an unpickled ``lock()`` from here problems could occur from this class not having atomic methods. """ def __init__(self): self.locked_status = False def acquire(self, waitflag=None): """Dummy implementation of acquire(). For blocking calls, self.locked_status is automatically set to True and returned appropriately based on value of ``waitflag``. If it is non-blocking, then the value is actually checked and not set if it is already acquired. This is all done so that threading.Condition's assert statements aren't triggered and throw a little fit. """ if waitflag is None or waitflag: self.locked_status = True return True else: if not self.locked_status: self.locked_status = True return True else: return False __enter__ = acquire def __exit__(self, typ, val, tb): self.release() def release(self): """Release the dummy lock.""" # XXX Perhaps shouldn't actually bother to test? Could lead # to problems for complex, threaded code. if not self.locked_status: raise error self.locked_status = False return True def locked(self): return self.locked_status # Used to signal that interrupt_main was called in a "thread" _interrupt = False # True when not executing in a "thread" _main = True def interrupt_main(): """Set _interrupt flag to True to have start_new_thread raise KeyboardInterrupt upon exiting.""" if _main: raise KeyboardInterrupt else: global _interrupt _interrupt = True ================================================ FILE: third_party/stdlib/fnmatch.py ================================================ """Filename matching with shell patterns. fnmatch(FILENAME, PATTERN) matches according to the local convention. fnmatchcase(FILENAME, PATTERN) always takes case in account. The functions operate by translating the pattern into a regular expression. They cache the compiled regular expressions for speed. The function translate(PATTERN) returns a regular expression corresponding to PATTERN. (It does not compile it.) """ import re __all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] _cache = {} _MAXCACHE = 100 def _purge(): """Clear the pattern cache""" # _cache.clear() globals()['_cache'] = {} def fnmatch(name, pat): """Test whether FILENAME matches PATTERN. Patterns are Unix shell style: * matches everything ? matches any single character [seq] matches any character in seq [!seq] matches any char not in seq An initial period in FILENAME is not special. Both FILENAME and PATTERN are first case-normalized if the operating system requires it. If you don't want this, use fnmatchcase(FILENAME, PATTERN). """ # import os # name = os.path.normcase(name) # pat = os.path.normcase(pat) return fnmatchcase(name, pat) def filter(names, pat): """Return the subset of the list NAMES that match PAT""" import os # import posixpath result=[] # pat=os.path.normcase(pat) try: re_pat = _cache[pat] except KeyError: res = translate(pat) if len(_cache) >= _MAXCACHE: # _cache.clear() globals()['_cache'] = {} _cache[pat] = re_pat = re.compile(res) match = re_pat.match # if os.path is posixpath: if 1: # normcase on posix is NOP. Optimize it away from the loop. for name in names: if match(name): result.append(name) else: for name in names: if match(os.path.normcase(name)): result.append(name) return result def fnmatchcase(name, pat): """Test whether FILENAME matches PATTERN, including case. This is a version of fnmatch() which doesn't case-normalize its arguments. """ try: re_pat = _cache[pat] except KeyError: res = translate(pat) if len(_cache) >= _MAXCACHE: # _cache.clear() globals()['_cache'] = {} _cache[pat] = re_pat = re.compile(res) return re_pat.match(name) is not None def translate(pat): """Translate a shell PATTERN to a regular expression. There is no way to quote meta-characters. """ i, n = 0, len(pat) res = '' while i < n: c = pat[i] i = i+1 if c == '*': res = res + '.*' elif c == '?': res = res + '.' elif c == '[': j = i if j < n and pat[j] == '!': j = j+1 if j < n and pat[j] == ']': j = j+1 while j < n and pat[j] != ']': j = j+1 if j >= n: res = res + '\\[' else: stuff = pat[i:j].replace('\\','\\\\') i = j+1 if stuff[0] == '!': stuff = '^' + stuff[1:] elif stuff[0] == '^': stuff = '\\' + stuff res = '%s[%s]' % (res, stuff) else: res = res + re.escape(c) return res + '\Z(?ms)' ================================================ FILE: third_party/stdlib/fpformat.py ================================================ """General floating point formatting functions. Functions: fix(x, digits_behind) sci(x, digits_behind) Each takes a number or a string and a number of digits as arguments. Parameters: x: number to be formatted; or a string resembling a number digits_behind: number of digits behind the decimal point """ from warnings import warnpy3k warnpy3k("the fpformat module has been removed in Python 3.0", stacklevel=2) del warnpy3k import re __all__ = ["fix","sci","NotANumber"] # Compiled regular expression to "decode" a number decoder = re.compile(r'^([-+]?)0*(\d*)((?:\.\d*)?)(([eE][-+]?\d+)?)$') # \0 the whole thing # \1 leading sign or empty # \2 digits left of decimal point # \3 fraction (empty or begins with point) # \4 exponent part (empty or begins with 'e' or 'E') try: class NotANumber(ValueError): pass except TypeError: NotANumber = 'fpformat.NotANumber' def extract(s): """Return (sign, intpart, fraction, expo) or raise an exception: sign is '+' or '-' intpart is 0 or more digits beginning with a nonzero fraction is 0 or more digits expo is an integer""" res = decoder.match(s) if res is None: raise NotANumber, s sign, intpart, fraction, exppart = res.group(1,2,3,4) if sign == '+': sign = '' if fraction: fraction = fraction[1:] if exppart: expo = int(exppart[1:]) else: expo = 0 return sign, intpart, fraction, expo def unexpo(intpart, fraction, expo): """Remove the exponent by changing intpart and fraction.""" if expo > 0: # Move the point left f = len(fraction) intpart, fraction = intpart + fraction[:expo], fraction[expo:] if expo > f: intpart = intpart + '0'*(expo-f) elif expo < 0: # Move the point right i = len(intpart) intpart, fraction = intpart[:expo], intpart[expo:] + fraction if expo < -i: fraction = '0'*(-expo-i) + fraction return intpart, fraction def roundfrac(intpart, fraction, digs): """Round or extend the fraction to size digs.""" f = len(fraction) if f <= digs: return intpart, fraction + '0'*(digs-f) i = len(intpart) if i+digs < 0: return '0'*-digs, '' total = intpart + fraction nextdigit = total[i+digs] if nextdigit >= '5': # Hard case: increment last digit, may have carry! n = i + digs - 1 while n >= 0: if total[n] != '9': break n = n-1 else: total = '0' + total i = i+1 n = 0 total = total[:n] + chr(ord(total[n]) + 1) + '0'*(len(total)-n-1) intpart, fraction = total[:i], total[i:] if digs >= 0: return intpart, fraction[:digs] else: return intpart[:digs] + '0'*-digs, '' def fix(x, digs): """Format x as [-]ddd.ddd with 'digs' digits after the point and at least one digit before. If digs <= 0, the point is suppressed.""" if type(x) != type(''): x = repr(x) try: sign, intpart, fraction, expo = extract(x) except NotANumber: return x intpart, fraction = unexpo(intpart, fraction, expo) intpart, fraction = roundfrac(intpart, fraction, digs) while intpart and intpart[0] == '0': intpart = intpart[1:] if intpart == '': intpart = '0' if digs > 0: return sign + intpart + '.' + fraction else: return sign + intpart def sci(x, digs): """Format x as [-]d.dddE[+-]ddd with 'digs' digits after the point and exactly one digit before. If digs is <= 0, one digit is kept and the point is suppressed.""" if type(x) != type(''): x = repr(x) sign, intpart, fraction, expo = extract(x) if not intpart: while fraction and fraction[0] == '0': fraction = fraction[1:] expo = expo - 1 if fraction: intpart, fraction = fraction[0], fraction[1:] expo = expo - 1 else: intpart = '0' else: expo = expo + len(intpart) - 1 intpart, fraction = intpart[0], intpart[1:] + fraction digs = max(0, digs) intpart, fraction = roundfrac(intpart, fraction, digs) if len(intpart) > 1: intpart, fraction, expo = \ intpart[0], intpart[1:] + fraction[:-1], \ expo + len(intpart) - 1 s = sign + intpart if digs > 0: s = s + '.' + fraction e = repr(abs(expo)) e = '0'*(3-len(e)) + e if expo < 0: e = '-' + e else: e = '+' + e return s + 'e' + e def test(): """Interactive test run.""" try: while 1: x, digs = input('Enter (x, digs): ') print x, fix(x, digs), sci(x, digs) except (EOFError, KeyboardInterrupt): pass ================================================ FILE: third_party/stdlib/functools.py ================================================ """functools.py - Tools for working with functions and callable objects """ # Python module wrapper for _functools C module # to allow utilities written in Python to be added # to the functools module. # Written by Nick Coghlan # Copyright (C) 2006 Python Software Foundation. # See C source code for _functools credits/copyright # from _functools import partial, reduce import _functools partial = _functools.partial reduce = _functools.reduce def setattr(d, k, v): d.__dict__[k] = v # update_wrapper() and wraps() are tools to help write # wrapper functions that can handle naive introspection WRAPPER_ASSIGNMENTS = ('__module__', '__name__') #, '__doc__' WRAPPER_UPDATES = ('__dict__',) def update_wrapper(wrapper, wrapped, assigned = WRAPPER_ASSIGNMENTS, updated = WRAPPER_UPDATES): """Update a wrapper function to look like the wrapped function wrapper is the function to be updated wrapped is the original function assigned is a tuple naming the attributes assigned directly from the wrapped function to the wrapper function (defaults to functools.WRAPPER_ASSIGNMENTS) updated is a tuple naming the attributes of the wrapper that are updated with the corresponding attribute from the wrapped function (defaults to functools.WRAPPER_UPDATES) """ for attr in assigned: setattr(wrapper, attr, getattr(wrapped, attr)) for attr in updated: getattr(wrapper, attr).update(getattr(wrapped, attr, {})) # Return the wrapper so this can be used as a decorator via partial() return wrapper def wraps(wrapped, assigned = WRAPPER_ASSIGNMENTS, updated = WRAPPER_UPDATES): """Decorator factory to apply update_wrapper() to a wrapper function Returns a decorator that invokes update_wrapper() with the decorated function as the wrapper argument and the arguments to wraps() as the remaining arguments. Default arguments are as for update_wrapper(). This is a convenience function to simplify applying partial() to update_wrapper(). """ return partial(update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated) def total_ordering(cls): """Class decorator that fills in missing ordering methods""" convert = { '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), ('__le__', lambda self, other: self < other or self == other), ('__ge__', lambda self, other: not self < other)], '__le__': [('__ge__', lambda self, other: not self <= other or self == other), ('__lt__', lambda self, other: self <= other and not self == other), ('__gt__', lambda self, other: not self <= other)], '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), ('__ge__', lambda self, other: self > other or self == other), ('__le__', lambda self, other: not self > other)], '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), ('__gt__', lambda self, other: self >= other and not self == other), ('__lt__', lambda self, other: not self >= other)] } roots = set(dir(cls)) & set(convert) if not roots: raise ValueError('must define at least one ordering operation: < > <= >=') root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ for opname, opfunc in convert[root]: if opname not in roots: opfunc.__name__ = opname opfunc.__doc__ = getattr(int, opname).__doc__ setattr(cls, opname, opfunc) return cls def cmp_to_key(mycmp): """Convert a cmp= function into a key= function""" class K(object): __slots__ = ['obj'] def __init__(self, obj, *args): self.obj = obj def __lt__(self, other): return mycmp(self.obj, other.obj) < 0 def __gt__(self, other): return mycmp(self.obj, other.obj) > 0 def __eq__(self, other): return mycmp(self.obj, other.obj) == 0 def __le__(self, other): return mycmp(self.obj, other.obj) <= 0 def __ge__(self, other): return mycmp(self.obj, other.obj) >= 0 def __ne__(self, other): return mycmp(self.obj, other.obj) != 0 def __hash__(self): raise TypeError('hash not implemented') return K ================================================ FILE: third_party/stdlib/genericpath.py ================================================ """ Path operations common to more than one OS Do not use directly. The OS specific modules import the appropriate functions from this module themselves. """ import os import stat __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', 'getsize', 'isdir', 'isfile'] try: _unicode = unicode except NameError: # If Python is built without Unicode support, the unicode type # will not exist. Fake one. class _unicode(object): pass # Does a path exist? # This is false for dangling symbolic links on systems that support them. def exists(path): """Test whether a path exists. Returns False for broken symbolic links""" try: os.stat(path) except os.error: return False return True # This follows symbolic links, so both islink() and isdir() can be true # for the same path on systems that support symlinks def isfile(path): """Test whether a path is a regular file""" try: st = os.stat(path) except os.error: return False return stat.S_ISREG(st.st_mode) # Is a path a directory? # This follows symbolic links, so both islink() and isdir() # can be true for the same path on systems that support symlinks def isdir(s): """Return true if the pathname refers to an existing directory.""" try: st = os.stat(s) except os.error: return False return stat.S_ISDIR(st.st_mode) def getsize(filename): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size def getmtime(filename): """Return the last modification time of a file, reported by os.stat().""" return os.stat(filename).st_mtime def getatime(filename): """Return the last access time of a file, reported by os.stat().""" return os.stat(filename).st_atime def getctime(filename): """Return the metadata change time of a file, reported by os.stat().""" return os.stat(filename).st_ctime # Return the longest prefix of all list elements. def commonprefix(m): "Given a list of pathnames, returns the longest common leading component" if not m: return '' s1 = min(m) s2 = max(m) for i, c in enumerate(s1): if c != s2[i]: return s1[:i] return s1 # Split a path in root and extension. # The extension is everything starting at the last dot in the last # pathname component; the root is everything before that. # It is always true that root + ext == p. # Generic implementation of splitext, to be parametrized with # the separators def _splitext(p, sep, altsep, extsep): """Split the extension from a pathname. Extension is everything from the last dot to the end, ignoring leading dots. Returns "(root, ext)"; ext may be empty.""" sepIndex = p.rfind(sep) if altsep: altsepIndex = p.rfind(altsep) sepIndex = max(sepIndex, altsepIndex) dotIndex = p.rfind(extsep) if dotIndex > sepIndex: # skip all leading dots filenameIndex = sepIndex + 1 while filenameIndex < dotIndex: if p[filenameIndex] != extsep: return p[:dotIndex], p[dotIndex:] filenameIndex += 1 return p, '' ================================================ FILE: third_party/stdlib/getopt.py ================================================ """Parser for command line options. This module helps scripts to parse the command line arguments in sys.argv. It supports the same conventions as the Unix getopt() function (including the special meanings of arguments of the form `-' and `--'). Long options similar to those supported by GNU software may be used as well via an optional third argument. This module provides two functions and an exception: getopt() -- Parse command line options gnu_getopt() -- Like getopt(), but allow option and non-option arguments to be intermixed. GetoptError -- exception (class) raised with 'opt' attribute, which is the option involved with the exception. """ # Long option support added by Lars Wirzenius . # # Gerrit Holl moved the string-based exceptions # to class-based exceptions. # # Peter Astrand added gnu_getopt(). # # TODO for gnu_getopt(): # # - GNU getopt_long_only mechanism # - allow the caller to specify ordering # - RETURN_IN_ORDER option # - GNU extension with '-' as first character of option string # - optional arguments, specified by double colons # - an option string with a W followed by semicolon should # treat "-W foo" as "--foo" __all__ = ["GetoptError","error","getopt","gnu_getopt"] import os class GetoptError(Exception): opt = '' msg = '' def __init__(self, msg, opt=''): self.msg = msg self.opt = opt Exception.__init__(self, msg, opt) def __str__(self): return self.msg error = GetoptError # backward compatibility def getopt(args, shortopts, longopts = []): """getopt(args, options[, long_options]) -> opts, args Parses command line options and parameter list. args is the argument list to be parsed, without the leading reference to the running program. Typically, this means "sys.argv[1:]". shortopts is the string of option letters that the script wants to recognize, with options that require an argument followed by a colon (i.e., the same format that Unix getopt() uses). If specified, longopts is a list of strings with the names of the long options which should be supported. The leading '--' characters should not be included in the option name. Options which require an argument should be followed by an equal sign ('='). The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments left after the option list was stripped (this is a trailing slice of the first argument). Each option-and-value pair returned has the option as its first element, prefixed with a hyphen (e.g., '-x'), and the option argument as its second element, or an empty string if the option has no argument. The options occur in the list in the same order in which they were found, thus allowing multiple occurrences. Long and short options may be mixed. """ opts = [] if type(longopts) == type(""): longopts = [longopts] else: longopts = list(longopts) while args and args[0].startswith('-') and args[0] != '-': if args[0] == '--': args = args[1:] break if args[0].startswith('--'): opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) else: opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) return opts, args def gnu_getopt(args, shortopts, longopts = []): """getopt(args, options[, long_options]) -> opts, args This function works like getopt(), except that GNU style scanning mode is used by default. This means that option and non-option arguments may be intermixed. The getopt() function stops processing options as soon as a non-option argument is encountered. If the first character of the option string is `+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. """ opts = [] prog_args = [] if isinstance(longopts, str): longopts = [longopts] else: longopts = list(longopts) # Allow options after non-option arguments? if shortopts.startswith('+'): shortopts = shortopts[1:] all_options_first = True elif os.environ.get("POSIXLY_CORRECT"): all_options_first = True else: all_options_first = False while args: if args[0] == '--': prog_args += args[1:] break if args[0][:2] == '--': opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) elif args[0][:1] == '-' and args[0] != '-': opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) else: if all_options_first: prog_args += args break else: prog_args.append(args[0]) args = args[1:] return opts, prog_args def do_longs(opts, opt, longopts, args): try: i = opt.index('=') except ValueError: optarg = None else: opt, optarg = opt[:i], opt[i+1:] has_arg, opt = long_has_args(opt, longopts) if has_arg: if optarg is None: if not args: raise GetoptError('option --%s requires argument' % opt, opt) optarg, args = args[0], args[1:] elif optarg is not None: raise GetoptError('option --%s must not have an argument' % opt, opt) opts.append(('--' + opt, optarg or '')) return opts, args # Return: # has_arg? # full option name def long_has_args(opt, longopts): possibilities = [o for o in longopts if o.startswith(opt)] if not possibilities: raise GetoptError('option --%s not recognized' % opt, opt) # Is there an exact match? if opt in possibilities: return False, opt elif opt + '=' in possibilities: return True, opt # No exact match, so better be unique. if len(possibilities) > 1: # XXX since possibilities contains all valid continuations, might be # nice to work them into the error msg raise GetoptError('option --%s not a unique prefix' % opt, opt) assert len(possibilities) == 1 unique_match = possibilities[0] has_arg = unique_match.endswith('=') if has_arg: unique_match = unique_match[:-1] return has_arg, unique_match def do_shorts(opts, optstring, shortopts, args): while optstring != '': opt, optstring = optstring[0], optstring[1:] if short_has_arg(opt, shortopts): if optstring == '': if not args: raise GetoptError('option -%s requires argument' % opt, opt) optstring, args = args[0], args[1:] optarg, optstring = optstring, '' else: optarg = '' opts.append(('-' + opt, optarg)) return opts, args def short_has_arg(opt, shortopts): for i in range(len(shortopts)): if opt == shortopts[i] != ':': return shortopts.startswith(':', i+1) raise GetoptError('option -%s not recognized' % opt, opt) if __name__ == '__main__': import sys print getopt(sys.argv[1:], "a:b", ["alpha=", "beta"]) ================================================ FILE: third_party/stdlib/glob.py ================================================ """Filename globbing utility.""" import sys import os import re import fnmatch try: _unicode = unicode except NameError: # If Python is built without Unicode support, the unicode type # will not exist. Fake one. class _unicode(object): pass __all__ = ["glob", "iglob"] def glob(pathname): """Return a list of paths matching a pathname pattern. The pattern may contain simple shell-style wildcards a la fnmatch. However, unlike fnmatch, filenames starting with a dot are special cases that are not matched by '*' and '?' patterns. """ return list(iglob(pathname)) def iglob(pathname): """Return an iterator which yields the paths matching a pathname pattern. The pattern may contain simple shell-style wildcards a la fnmatch. However, unlike fnmatch, filenames starting with a dot are special cases that are not matched by '*' and '?' patterns. """ dirname, basename = os.path.split(pathname) if not has_magic(pathname): if basename: if os.path.lexists(pathname): yield pathname else: # Patterns ending with a slash should match only directories if os.path.isdir(dirname): yield pathname return if not dirname: for name in glob1(os.curdir, basename): yield name return # `os.path.split()` returns the argument itself as a dirname if it is a # drive or UNC path. Prevent an infinite recursion if a drive or UNC path # contains magic characters (i.e. r'\\?\C:'). if dirname != pathname and has_magic(dirname): dirs = iglob(dirname) else: dirs = [dirname] if has_magic(basename): glob_in_dir = glob1 else: glob_in_dir = glob0 for dirname in dirs: for name in glob_in_dir(dirname, basename): yield os.path.join(dirname, name) # These 2 helper functions non-recursively glob inside a literal directory. # They return a list of basenames. `glob1` accepts a pattern while `glob0` # takes a literal basename (so it only has to check for its existence). def glob1(dirname, pattern): if not dirname: dirname = os.curdir if isinstance(pattern, _unicode) and not isinstance(dirname, unicode): dirname = unicode(dirname, sys.getfilesystemencoding() or sys.getdefaultencoding()) try: names = os.listdir(dirname) except os.error: return [] if pattern[0] != '.': # names = filter(lambda x: x[0] != '.', names) names = [x for x in names if x[0] != '.'] return fnmatch.filter(names, pattern) def glob0(dirname, basename): if basename == '': # `os.path.split()` returns an empty basename for paths ending with a # directory separator. 'q*x/' should match only directories. if os.path.isdir(dirname): return [basename] else: if os.path.lexists(os.path.join(dirname, basename)): return [basename] return [] magic_check = re.compile('[*?[]') def has_magic(s): return magic_check.search(s) is not None ================================================ FILE: third_party/stdlib/heapq.py ================================================ # -*- coding: latin-1 -*- """Heap queue algorithm (a.k.a. priority queue). Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for all k, counting elements from 0. For the sake of comparison, non-existing elements are considered to be infinite. The interesting property of a heap is that a[0] is always its smallest element. Usage: heap = [] # creates an empty heap heappush(heap, item) # pushes a new item on the heap item = heappop(heap) # pops the smallest item from the heap item = heap[0] # smallest item on the heap without popping it heapify(x) # transforms list into a heap, in-place, in linear time item = heapreplace(heap, item) # pops and returns smallest item, and adds # new item; the heap size is unchanged Our API differs from textbook heap algorithms as follows: - We use 0-based indexing. This makes the relationship between the index for a node and the indexes for its children slightly less obvious, but is more suitable since Python uses 0-based indexing. - Our heappop() method returns the smallest item, not the largest. These two make it possible to view the heap as a regular Python list without surprises: heap[0] is the smallest item, and heap.sort() maintains the heap invariant! """ # Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger __about__ = """Heap queues [explanation by Franois Pinard] Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for all k, counting elements from 0. For the sake of comparison, non-existing elements are considered to be infinite. The interesting property of a heap is that a[0] is always its smallest element. The strange invariant above is meant to be an efficient memory representation for a tournament. The numbers below are `k', not a[k]: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In a usual binary tournament we see in sports, each cell is the winner over the two cells it tops, and we can trace the winner down the tree to see all opponents s/he had. However, in many computer applications of such tournaments, we do not need to trace the history of a winner. To be more memory efficient, when a winner is promoted, we try to replace it by something else at a lower level, and the rule becomes that a cell and the two cells it tops contain three different items, but the top cell "wins" over the two topped cells. If this heap invariant is protected at all time, index 0 is clearly the overall winner. The simplest algorithmic way to remove it and find the "next" winner is to move some loser (let's say cell 30 in the diagram above) into the 0 position, and then percolate this new 0 down the tree, exchanging values, until the invariant is re-established. This is clearly logarithmic on the total number of items in the tree. By iterating over all items, you get an O(n ln n) sort. A nice feature of this sort is that you can efficiently insert new items while the sort is going on, provided that the inserted items are not "better" than the last 0'th element you extracted. This is especially useful in simulation contexts, where the tree holds all incoming events, and the "win" condition means the smallest scheduled time. When an event schedule other events for execution, they are scheduled into the future, so they can easily go into the heap. So, a heap is a good structure for implementing schedulers (this is what I used for my MIDI sequencer :-). Various structures for implementing schedulers have been extensively studied, and heaps are good for this, as they are reasonably speedy, the speed is almost constant, and the worst case is not much different than the average case. However, there are other representations which are more efficient overall, yet the worst cases might be terrible. Heaps are also very useful in big disk sorts. You most probably all know that a big sort implies producing "runs" (which are pre-sorted sequences, which size is usually related to the amount of CPU memory), followed by a merging passes for these runs, which merging is often very cleverly organised[1]. It is very important that the initial sort produces the longest runs possible. Tournaments are a good way to that. If, using all the memory available to hold a tournament, you replace and percolate items that happen to fit the current run, you'll produce runs which are twice the size of the memory for random input, and much better for input fuzzily ordered. Moreover, if you output the 0'th item on disk and get an input which may not fit in the current tournament (because the value "wins" over the last output value), it cannot fit in the heap, so the size of the heap decreases. The freed memory could be cleverly reused immediately for progressively building a second heap, which grows at exactly the same rate the first heap is melting. When the first heap completely vanishes, you switch heaps and start a new run. Clever and quite effective! In a word, heaps are useful memory structures to know. I use them in a few applications, and I think it is good to keep a `heap' module around. :-) -------------------- [1] The disk balancing algorithms which are current, nowadays, are more annoying than clever, and this is a consequence of the seeking capabilities of the disks. On devices which cannot seek, like big tape drives, the story was quite different, and one had to be very clever to ensure (far in advance) that each tape movement will be the most effective possible (that is, will best participate at "progressing" the merge). Some tapes were even able to read backwards, and this was also used to avoid the rewinding time. Believe me, real good tape sorts were quite spectacular to watch! From all times, sorting has always been a Great Art! :-) """ __all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', 'nlargest', 'nsmallest', 'heappushpop'] import itertools islice = itertools.islice count = itertools.count imap = itertools.imap izip = itertools.izip tee = itertools.tee chain = itertools.chain import operator itemgetter = operator.itemgetter def cmp_lt(x, y): # Use __lt__ if available; otherwise, try __le__. # In Py3.x, only __lt__ will be called. return (x < y) if hasattr(x, '__lt__') else (not y <= x) def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" heap.append(item) _siftdown(heap, 0, len(heap)-1) def heappop(heap): """Pop the smallest item off the heap, maintaining the heap invariant.""" lastelt = heap.pop() # raises appropriate IndexError if heap is empty if heap: returnitem = heap[0] heap[0] = lastelt _siftup(heap, 0) else: returnitem = lastelt return returnitem def heapreplace(heap, item): """Pop and return the current smallest value, and add the new item. This is more efficient than heappop() followed by heappush(), and can be more appropriate when using a fixed-size heap. Note that the value returned may be larger than item! That constrains reasonable uses of this routine unless written as part of a conditional replacement: if item > heap[0]: item = heapreplace(heap, item) """ returnitem = heap[0] # raises appropriate IndexError if heap is empty heap[0] = item _siftup(heap, 0) return returnitem def heappushpop(heap, item): """Fast version of a heappush followed by a heappop.""" if heap and cmp_lt(heap[0], item): item, heap[0] = heap[0], item _siftup(heap, 0) return item def heapify(x): """Transform list into a heap, in-place, in O(len(x)) time.""" n = len(x) # Transform bottom-up. The largest index there's any point to looking at # is the largest with a child index in-range, so must have 2*i + 1 < n, # or i < (n-1)/2. If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so # j-1 is the largest, which is n//2 - 1. If n is odd = 2*j+1, this is # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1. for i in reversed(xrange(n//2)): _siftup(x, i) def _heappushpop_max(heap, item): """Maxheap version of a heappush followed by a heappop.""" if heap and cmp_lt(item, heap[0]): item, heap[0] = heap[0], item _siftup_max(heap, 0) return item def _heapify_max(x): """Transform list into a maxheap, in-place, in O(len(x)) time.""" n = len(x) for i in reversed(range(n//2)): _siftup_max(x, i) def nlargest(n, iterable): """Find the n largest elements in a dataset. Equivalent to: sorted(iterable, reverse=True)[:n] """ if n < 0: return [] it = iter(iterable) result = list(islice(it, n)) if not result: return result heapify(result) _heappushpop = heappushpop for elem in it: _heappushpop(result, elem) result.sort(reverse=True) return result def nsmallest(n, iterable): """Find the n smallest elements in a dataset. Equivalent to: sorted(iterable)[:n] """ if n < 0: return [] it = iter(iterable) result = list(islice(it, n)) if not result: return result _heapify_max(result) _heappushpop = _heappushpop_max for elem in it: _heappushpop(result, elem) result.sort() return result # 'heap' is a heap at all indices >= startpos, except possibly for pos. pos # is the index of a leaf with a possibly out-of-order value. Restore the # heap invariant. def _siftdown(heap, startpos, pos): newitem = heap[pos] # Follow the path to the root, moving parents down until finding a place # newitem fits. while pos > startpos: parentpos = (pos - 1) >> 1 parent = heap[parentpos] if cmp_lt(newitem, parent): heap[pos] = parent pos = parentpos continue break heap[pos] = newitem # The child indices of heap index pos are already heaps, and we want to make # a heap at index pos too. We do this by bubbling the smaller child of # pos up (and so on with that child's children, etc) until hitting a leaf, # then using _siftdown to move the oddball originally at index pos into place. # # We *could* break out of the loop as soon as we find a pos where newitem <= # both its children, but turns out that's not a good idea, and despite that # many books write the algorithm that way. During a heap pop, the last array # element is sifted in, and that tends to be large, so that comparing it # against values starting from the root usually doesn't pay (= usually doesn't # get us out of the loop early). See Knuth, Volume 3, where this is # explained and quantified in an exercise. # # Cutting the # of comparisons is important, since these routines have no # way to extract "the priority" from an array element, so that intelligence # is likely to be hiding in custom __cmp__ methods, or in array elements # storing (priority, record) tuples. Comparisons are thus potentially # expensive. # # On random arrays of length 1000, making this change cut the number of # comparisons made by heapify() a little, and those made by exhaustive # heappop() a lot, in accord with theory. Here are typical results from 3 # runs (3 just to demonstrate how small the variance is): # # Compares needed by heapify Compares needed by 1000 heappops # -------------------------- -------------------------------- # 1837 cut to 1663 14996 cut to 8680 # 1855 cut to 1659 14966 cut to 8678 # 1847 cut to 1660 15024 cut to 8703 # # Building the heap by using heappush() 1000 times instead required # 2198, 2148, and 2219 compares: heapify() is more efficient, when # you can use it. # # The total compares needed by list.sort() on the same lists were 8627, # 8627, and 8632 (this should be compared to the sum of heapify() and # heappop() compares): list.sort() is (unsurprisingly!) more efficient # for sorting. def _siftup(heap, pos): endpos = len(heap) startpos = pos newitem = heap[pos] # Bubble up the smaller child until hitting a leaf. childpos = 2*pos + 1 # leftmost child position while childpos < endpos: # Set childpos to index of smaller child. rightpos = childpos + 1 if rightpos < endpos and not cmp_lt(heap[childpos], heap[rightpos]): childpos = rightpos # Move the smaller child up. heap[pos] = heap[childpos] pos = childpos childpos = 2*pos + 1 # The leaf at pos is empty now. Put newitem there, and bubble it up # to its final resting place (by sifting its parents down). heap[pos] = newitem _siftdown(heap, startpos, pos) def _siftdown_max(heap, startpos, pos): 'Maxheap variant of _siftdown' newitem = heap[pos] # Follow the path to the root, moving parents down until finding a place # newitem fits. while pos > startpos: parentpos = (pos - 1) >> 1 parent = heap[parentpos] if cmp_lt(parent, newitem): heap[pos] = parent pos = parentpos continue break heap[pos] = newitem def _siftup_max(heap, pos): 'Maxheap variant of _siftup' endpos = len(heap) startpos = pos newitem = heap[pos] # Bubble up the larger child until hitting a leaf. childpos = 2*pos + 1 # leftmost child position while childpos < endpos: # Set childpos to index of larger child. rightpos = childpos + 1 if rightpos < endpos and not cmp_lt(heap[rightpos], heap[childpos]): childpos = rightpos # Move the larger child up. heap[pos] = heap[childpos] pos = childpos childpos = 2*pos + 1 # The leaf at pos is empty now. Put newitem there, and bubble it up # to its final resting place (by sifting its parents down). heap[pos] = newitem _siftdown_max(heap, startpos, pos) # If available, use C implementation #try: # import _heapq #except ImportError: # pass def merge(*iterables): '''Merge multiple sorted inputs into a single sorted output. Similar to sorted(itertools.chain(*iterables)) but returns a generator, does not pull the data into memory all at once, and assumes that each of the input streams is already sorted (smallest to largest). >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25])) [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25] ''' _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration _len = len h = [] h_append = h.append for itnum, it in enumerate(map(iter, iterables)): try: next = it.next h_append([next(), itnum, next]) except _StopIteration: pass heapify(h) while _len(h) > 1: try: while 1: v, itnum, next = s = h[0] yield v s[0] = next() # raises StopIteration when exhausted _heapreplace(h, s) # restore heap condition except _StopIteration: _heappop(h) # remove empty iterator if h: # fast case when only a single iterator remains v, itnum, next = h[0] yield v for v in next.__self__: yield v # Extend the implementations of nsmallest and nlargest to use a key= argument _nsmallest = nsmallest def nsmallest(n, iterable, key=None): """Find the n smallest elements in a dataset. Equivalent to: sorted(iterable, key=key)[:n] """ # Short-cut for n==1 is to use min() when len(iterable)>0 if n == 1: it = iter(iterable) head = list(islice(it, 1)) if not head: return [] if key is None: return [min(chain(head, it))] return [min(chain(head, it), key=key)] # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): pass else: if n >= size: return sorted(iterable, key=key)[:n] # When key is none, use simpler decoration if key is None: it = izip(iterable, count()) # decorate result = _nsmallest(n, it) return map(itemgetter(0), result) # undecorate # General case, slowest method in1, in2 = tee(iterable) it = izip(imap(key, in1), count(), in2) # decorate result = _nsmallest(n, it) return map(itemgetter(2), result) # undecorate _nlargest = nlargest def nlargest(n, iterable, key=None): """Find the n largest elements in a dataset. Equivalent to: sorted(iterable, key=key, reverse=True)[:n] """ # Short-cut for n==1 is to use max() when len(iterable)>0 if n == 1: it = iter(iterable) head = list(islice(it, 1)) if not head: return [] if key is None: return [max(chain(head, it))] return [max(chain(head, it), key=key)] # When n>=size, it's faster to use sorted() try: size = len(iterable) except (TypeError, AttributeError): pass else: if n >= size: return sorted(iterable, key=key, reverse=True)[:n] # When key is none, use simpler decoration if key is None: it = izip(iterable, count(0,-1)) # decorate result = _nlargest(n, it) return map(itemgetter(0), result) # undecorate # General case, slowest method in1, in2 = tee(iterable) it = izip(imap(key, in1), count(0,-1), in2) # decorate result = _nlargest(n, it) return map(itemgetter(2), result) # undecorate #if __name__ == "__main__": # # Simple sanity test # heap = [] # data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] # for item in data: # heappush(heap, item) # sort = [] # while heap: # sort.append(heappop(heap)) # print sort # # import doctest # doctest.testmod() ================================================ FILE: third_party/stdlib/json/__init__.py ================================================ r"""JSON (JavaScript Object Notation) is a subset of JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. :mod:`json` exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. It is the externally maintained version of the :mod:`json` library contained in Python 2.6, but maintains compatibility with Python 2.4 and Python 2.5 and (currently) has significant performance advantages, even without using the optional C extension for speedups. Encoding basic Python object hierarchies:: >>> import json >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) '["foo", {"bar": ["baz", null, 1.0, 2]}]' >>> print json.dumps("\"foo\bar") "\"foo\bar" >>> print json.dumps(u'\u1234') "\u1234" >>> print json.dumps('\\') "\\" >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) {"a": 0, "b": 0, "c": 0} >>> from StringIO import StringIO >>> io = StringIO() >>> json.dump(['streaming API'], io) >>> io.getvalue() '["streaming API"]' Compact encoding:: >>> import json >>> json.dumps([1,2,3,{'4': 5, '6': 7}], sort_keys=True, separators=(',',':')) '[1,2,3,{"4":5,"6":7}]' Pretty printing:: >>> import json >>> print json.dumps({'4': 5, '6': 7}, sort_keys=True, ... indent=4, separators=(',', ': ')) { "4": 5, "6": 7 } Decoding JSON:: >>> import json >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj True >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' True >>> from StringIO import StringIO >>> io = StringIO('["streaming API"]') >>> json.load(io)[0] == 'streaming API' True Specializing JSON object decoding:: >>> import json >>> def as_complex(dct): ... if '__complex__' in dct: ... return complex(dct['real'], dct['imag']) ... return dct ... >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', ... object_hook=as_complex) (1+2j) >>> from decimal import Decimal >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') True Specializing JSON object encoding:: >>> import json >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... raise TypeError(repr(o) + " is not JSON serializable") ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) '[2.0, 1.0]' >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) '[2.0, 1.0]' Using json.tool from the shell to validate and pretty-print:: $ echo '{"json":"obj"}' | python -m json.tool { "json": "obj" } $ echo '{ 1.2:3.4}' | python -m json.tool Expecting property name enclosed in double quotes: line 1 column 3 (char 2) """ __version__ = '2.0.9' __all__ = [ 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONEncoder', ] __author__ = 'Bob Ippolito ' # from .decoder import JSONDecoder # from .encoder import JSONEncoder import json.decoder import json.encoder import json_scanner JSONDecoder = json.decoder.JSONDecoder JSONEncoder = json.encoder.JSONEncoder scanner = json_scanner _default_encoder = JSONEncoder( skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, indent=None, separators=None, encoding='utf-8', default=None, ) def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, encoding='utf-8', default=None, sort_keys=False, **kw): """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a ``.write()``-supporting file-like object). If ``skipkeys`` is true then ``dict`` keys that are not basic types (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is true (the default), all non-ASCII characters in the output are escaped with ``\uXXXX`` sequences, and the result is a ``str`` instance consisting of ASCII characters only. If ``ensure_ascii`` is ``False``, some chunks written to ``fp`` may be ``unicode`` instances. This usually happens because the input contains unicode strings or the ``encoding`` parameter is used. Unless ``fp.write()`` explicitly understands ``unicode`` (as in ``codecs.getwriter``) this is likely to cause an error. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. ``None`` is the most compact representation. Since the default item separator is ``', '``, the output might include trailing whitespace when ``indent`` is specified. You can use ``separators=(',', ': ')`` to avoid this. If ``separators`` is an ``(item_separator, dict_separator)`` tuple then it will be used instead of the default ``(', ', ': ')`` separators. ``(',', ':')`` is the most compact JSON representation. ``encoding`` is the character encoding for str instances, default is UTF-8. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *sort_keys* is ``True`` (default: ``False``), then the output of dictionaries will be sorted by key. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. """ # cached encoder if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and encoding == 'utf-8' and default is None and not sort_keys and not kw): iterable = _default_encoder.iterencode(obj) else: if cls is None: cls = JSONEncoder iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, separators=separators, encoding=encoding, default=default, sort_keys=sort_keys, **kw).iterencode(obj) # could accelerate with writelines in some versions of Python, at # a debuggability cost for chunk in iterable: fp.write(chunk) def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, encoding='utf-8', default=None, sort_keys=False, **kw): """Serialize ``obj`` to a JSON formatted ``str``. If ``skipkeys`` is true then ``dict`` keys that are not basic types (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is false, all non-ASCII characters are not escaped, and the return value may be a ``unicode`` instance. See ``dump`` for details. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. ``None`` is the most compact representation. Since the default item separator is ``', '``, the output might include trailing whitespace when ``indent`` is specified. You can use ``separators=(',', ': ')`` to avoid this. If ``separators`` is an ``(item_separator, dict_separator)`` tuple then it will be used instead of the default ``(', ', ': ')`` separators. ``(',', ':')`` is the most compact JSON representation. ``encoding`` is the character encoding for str instances, default is UTF-8. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *sort_keys* is ``True`` (default: ``False``), then the output of dictionaries will be sorted by key. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. """ # cached encoder if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and encoding == 'utf-8' and default is None and not sort_keys and not kw): return _default_encoder.encode(obj) if cls is None: cls = JSONEncoder return cls( skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, separators=separators, encoding=encoding, default=default, sort_keys=sort_keys, **kw).encode(obj) _default_decoder = JSONDecoder(encoding=None, object_hook=None, object_pairs_hook=None) def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw): """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing a JSON document) to a Python object. If the contents of ``fp`` is encoded with an ASCII based encoding other than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must be specified. Encodings that are not ASCII based (such as UCS-2) are not allowed, and should be wrapped with ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode`` object and passed to ``loads()`` ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). ``object_pairs_hook`` is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. """ return loads(fp.read(), encoding=encoding, cls=cls, object_hook=object_hook, parse_float=parse_float, parse_int=parse_int, parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw) def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw): """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON document) to a Python object. If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name must be specified. Encodings that are not ASCII based (such as UCS-2) are not allowed and should be decoded to ``unicode`` first. ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). ``object_pairs_hook`` is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser for JSON floats (e.g. decimal.Decimal). ``parse_int``, if specified, will be called with the string of every JSON int to be decoded. By default this is equivalent to int(num_str). This can be used to use another datatype or parser for JSON integers (e.g. float). ``parse_constant``, if specified, will be called with one of the following strings: -Infinity, Infinity, NaN, null, true, false. This can be used to raise an exception if invalid JSON numbers are encountered. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. """ if (cls is None and encoding is None and object_hook is None and parse_int is None and parse_float is None and parse_constant is None and object_pairs_hook is None and not kw): return _default_decoder.decode(s) if cls is None: cls = JSONDecoder if object_hook is not None: kw['object_hook'] = object_hook if object_pairs_hook is not None: kw['object_pairs_hook'] = object_pairs_hook if parse_float is not None: kw['parse_float'] = parse_float if parse_int is not None: kw['parse_int'] = parse_int if parse_constant is not None: kw['parse_constant'] = parse_constant return cls(encoding=encoding, **kw).decode(s) ================================================ FILE: third_party/stdlib/json/decoder.py ================================================ """Implementation of JSONDecoder """ import re import sys import _struct as struct # from json import scanner import json_scanner as scanner # try: # from _json import scanstring as c_scanstring # except ImportError: # c_scanstring = None c_scanstring = None __all__ = ['JSONDecoder'] FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL def _floatconstants(): nan, = struct.unpack('>d', b'\x7f\xf8\x00\x00\x00\x00\x00\x00') inf, = struct.unpack('>d', b'\x7f\xf0\x00\x00\x00\x00\x00\x00') return nan, inf, -inf NaN, PosInf, NegInf = _floatconstants() def linecol(doc, pos): lineno = doc.count('\n', 0, pos) + 1 if lineno == 1: colno = pos + 1 else: colno = pos - doc.rindex('\n', 0, pos) return lineno, colno def errmsg(msg, doc, pos, end=None): # Note that this function is called from _json lineno, colno = linecol(doc, pos) if end is None: # fmt = '{0}: line {1} column {2} (char {3})' # return fmt.format(msg, lineno, colno, pos) fmt = '%s: line %d column %d (char %d)' return fmt % (msg, lineno, colno, pos) endlineno, endcolno = linecol(doc, end) # fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' # return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) _CONSTANTS = { '-Infinity': NegInf, 'Infinity': PosInf, 'NaN': NaN, } STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) BACKSLASH = { '"': u'"', '\\': u'\\', '/': u'/', 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', } DEFAULT_ENCODING = "utf-8" def _decode_uXXXX(s, pos): esc = s[pos + 1:pos + 5] if len(esc) == 4 and esc[1] not in 'xX': try: return int(esc, 16) except ValueError: pass msg = "Invalid \\uXXXX escape" raise ValueError(errmsg(msg, s, pos)) def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match): """Scan the string s for a JSON string. End is the index of the character in s after the quote that started the JSON string. Unescapes all valid JSON string escape sequences and raises ValueError on attempt to decode an invalid string. If strict is False then literal control characters are allowed in the string. Returns a tuple of the decoded string and the index of the character in s after the end quote.""" if encoding is None: encoding = DEFAULT_ENCODING chunks = [] _append = chunks.append begin = end - 1 while 1: chunk = _m(s, end) if chunk is None: raise ValueError( errmsg("Unterminated string starting at", s, begin)) end = chunk.end() content, terminator = chunk.groups() # Content is contains zero or more unescaped string characters if content: if not isinstance(content, unicode): content = unicode(content, encoding) _append(content) # Terminator is the end of string, a literal control character, # or a backslash denoting that an escape sequence follows if terminator == '"': break elif terminator != '\\': if strict: msg = "Invalid control character %r at" % (terminator,) # msg = "Invalid control character {0!r} at".format(terminator) raise ValueError(errmsg(msg, s, end)) else: _append(terminator) continue try: esc = s[end] except IndexError: raise ValueError( errmsg("Unterminated string starting at", s, begin)) # If not a unicode escape sequence, must be in the lookup table if esc != 'u': try: char = _b[esc] except KeyError: msg = "Invalid \\escape: " + repr(esc) raise ValueError(errmsg(msg, s, end)) end += 1 else: # Unicode escape sequence uni = _decode_uXXXX(s, end) end += 5 # Check for surrogate pair on UCS-4 systems if sys.maxunicode > 65535 and \ 0xd800 <= uni <= 0xdbff and s[end:end + 2] == '\\u': uni2 = _decode_uXXXX(s, end + 1) if 0xdc00 <= uni2 <= 0xdfff: uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) end += 6 char = unichr(uni) # Append the unescaped character _append(char) return u''.join(chunks), end # Use speedup if available scanstring = c_scanstring or py_scanstring WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) WHITESPACE_STR = ' \t\n\r' def JSONObject(s_and_end, encoding, strict, scan_once, object_hook, object_pairs_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR): s, end = s_and_end pairs = [] pairs_append = pairs.append # Use a slice to prevent IndexError from being raised, the following # check will raise a more specific ValueError if the string is empty nextchar = s[end:end + 1] # Normally we expect nextchar == '"' if nextchar != '"': if nextchar in _ws: end = _w(s, end).end() nextchar = s[end:end + 1] # Trivial empty object if nextchar == '}': if object_pairs_hook is not None: result = object_pairs_hook(pairs) return result, end + 1 pairs = {} if object_hook is not None: pairs = object_hook(pairs) return pairs, end + 1 elif nextchar != '"': raise ValueError(errmsg( "Expecting property name enclosed in double quotes", s, end)) end += 1 while True: key, end = scanstring(s, end, encoding, strict) # To skip some function call overhead we optimize the fast paths where # the JSON key separator is ": " or just ":". if s[end:end + 1] != ':': end = _w(s, end).end() if s[end:end + 1] != ':': raise ValueError(errmsg("Expecting ':' delimiter", s, end)) end += 1 try: if s[end] in _ws: end += 1 if s[end] in _ws: end = _w(s, end + 1).end() except IndexError: pass try: value, end = scan_once(s, end) except StopIteration: raise ValueError(errmsg("Expecting object", s, end)) pairs_append((key, value)) try: nextchar = s[end] if nextchar in _ws: end = _w(s, end + 1).end() nextchar = s[end] except IndexError: nextchar = '' end += 1 if nextchar == '}': break elif nextchar != ',': raise ValueError(errmsg("Expecting ',' delimiter", s, end - 1)) try: nextchar = s[end] if nextchar in _ws: end += 1 nextchar = s[end] if nextchar in _ws: end = _w(s, end + 1).end() nextchar = s[end] except IndexError: nextchar = '' end += 1 if nextchar != '"': raise ValueError(errmsg( "Expecting property name enclosed in double quotes", s, end - 1)) if object_pairs_hook is not None: result = object_pairs_hook(pairs) return result, end pairs = dict(pairs) if object_hook is not None: pairs = object_hook(pairs) return pairs, end def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): s, end = s_and_end values = [] nextchar = s[end:end + 1] if nextchar in _ws: end = _w(s, end + 1).end() nextchar = s[end:end + 1] # Look-ahead for trivial empty array if nextchar == ']': return values, end + 1 _append = values.append while True: try: value, end = scan_once(s, end) except StopIteration: raise ValueError(errmsg("Expecting object", s, end)) _append(value) nextchar = s[end:end + 1] if nextchar in _ws: end = _w(s, end + 1).end() nextchar = s[end:end + 1] end += 1 if nextchar == ']': break elif nextchar != ',': raise ValueError(errmsg("Expecting ',' delimiter", s, end)) try: if s[end] in _ws: end += 1 if s[end] in _ws: end = _w(s, end + 1).end() except IndexError: pass return values, end class JSONDecoder(object): """Simple JSON decoder Performs the following translations in decoding by default: +---------------+-------------------+ | JSON | Python | +===============+===================+ | object | dict | +---------------+-------------------+ | array | list | +---------------+-------------------+ | string | unicode | +---------------+-------------------+ | number (int) | int, long | +---------------+-------------------+ | number (real) | float | +---------------+-------------------+ | true | True | +---------------+-------------------+ | false | False | +---------------+-------------------+ | null | None | +---------------+-------------------+ It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their corresponding ``float`` values, which is outside the JSON spec. """ def __init__(self, encoding=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None): """``encoding`` determines the encoding used to interpret any ``str`` objects decoded by this instance (utf-8 by default). It has no effect when decoding ``unicode`` objects. Note that currently only encodings that are a superset of ASCII work, strings of other encodings should be passed in as ``unicode``. ``object_hook``, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). ``object_pairs_hook``, if specified will be called with the result of every JSON object decoded with an ordered list of pairs. The return value of ``object_pairs_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, collections.OrderedDict will remember the order of insertion). If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to float(num_str). This can be used to use another datatype or parser for JSON floats (e.g. decimal.Decimal). ``parse_int``, if specified, will be called with the string of every JSON int to be decoded. By default this is equivalent to int(num_str). This can be used to use another datatype or parser for JSON integers (e.g. float). ``parse_constant``, if specified, will be called with one of the following strings: -Infinity, Infinity, NaN. This can be used to raise an exception if invalid JSON numbers are encountered. If ``strict`` is false (true is the default), then control characters will be allowed inside strings. Control characters in this context are those with character codes in the 0-31 range, including ``'\\t'`` (tab), ``'\\n'``, ``'\\r'`` and ``'\\0'``. """ self.encoding = encoding self.object_hook = object_hook self.object_pairs_hook = object_pairs_hook self.parse_float = parse_float or float self.parse_int = parse_int or int self.parse_constant = parse_constant or _CONSTANTS.__getitem__ self.strict = strict self.parse_object = JSONObject self.parse_array = JSONArray self.parse_string = scanstring self.scan_once = scanner.make_scanner(self) def decode(self, s, _w=WHITESPACE.match): """Return the Python representation of ``s`` (a ``str`` or ``unicode`` instance containing a JSON document) """ obj, end = self.raw_decode(s, idx=_w(s, 0).end()) end = _w(s, end).end() if end != len(s): raise ValueError(errmsg("Extra data", s, end, len(s))) return obj def raw_decode(self, s, idx=0): """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning with a JSON document) and return a 2-tuple of the Python representation and the index in ``s`` where the document ended. This can be used to decode a JSON document from a string that may have extraneous data at the end. """ try: obj, end = self.scan_once(s, idx) except StopIteration: raise ValueError("No JSON object could be decoded") return obj, end ================================================ FILE: third_party/stdlib/json/encoder.py ================================================ """Implementation of JSONEncoder """ import re # try: # from _json import encode_basestring_ascii as c_encode_basestring_ascii # except ImportError: # c_encode_basestring_ascii = None c_encode_basestring_ascii = None # try: # from _json import make_encoder as c_make_encoder # except ImportError: # c_make_encoder = None c_make_encoder = None def x4(i): return ("000%x" % i)[-4:] ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') HAS_UTF8 = re.compile(r'[\x80-\xff]') ESCAPE_DCT = { '\\': '\\\\', '"': '\\"', '\b': '\\b', '\f': '\\f', '\n': '\\n', '\r': '\\r', '\t': '\\t', } for i in range(0x20): # ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) # ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) # ESCAPE_DCT.setdefault(chr(i), '\\u' + x4(i)) ESCAPE_DCT[chr(i)] = '\\u' + x4(i) INFINITY = float('inf') FLOAT_REPR = repr def encode_basestring(s): """Return a JSON representation of a Python string """ def replace(match): return ESCAPE_DCT[match.group(0)] return '"' + ESCAPE.sub(replace, s) + '"' def py_encode_basestring_ascii(s): """Return an ASCII-only JSON representation of a Python string """ if isinstance(s, str) and HAS_UTF8.search(s) is not None: s = s.decode('utf-8') def replace(match): s = match.group(0) try: return ESCAPE_DCT[s] except KeyError: n = ord(s) if n < 0x10000: # return '\\u{0:04x}'.format(n) #return '\\u%04x' % (n,) return '\\u' + x4(n) else: # surrogate pair n -= 0x10000 s1 = 0xd800 | ((n >> 10) & 0x3ff) s2 = 0xdc00 | (n & 0x3ff) # return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) #return '\\u%04x\\u%04x' % (s1, s2) return '\\u' + x4(s1) + '\\u' + x4(s2) return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' encode_basestring_ascii = ( c_encode_basestring_ascii or py_encode_basestring_ascii) class JSONEncoder(object): """Extensible JSON encoder for Python data structures. Supports the following objects and types by default: +-------------------+---------------+ | Python | JSON | +===================+===============+ | dict | object | +-------------------+---------------+ | list, tuple | array | +-------------------+---------------+ | str, unicode | string | +-------------------+---------------+ | int, long, float | number | +-------------------+---------------+ | True | true | +-------------------+---------------+ | False | false | +-------------------+---------------+ | None | null | +-------------------+---------------+ To extend this to recognize other objects, subclass and implement a ``.default()`` method with another method that returns a serializable object for ``o`` if possible, otherwise it should call the superclass implementation (to raise ``TypeError``). """ item_separator = ', ' key_separator = ': ' def __init__(self, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, sort_keys=False, indent=None, separators=None, encoding='utf-8', default=None): """Constructor for JSONEncoder, with sensible defaults. If skipkeys is false, then it is a TypeError to attempt encoding of keys that are not str, int, long, float or None. If skipkeys is True, such items are simply skipped. If *ensure_ascii* is true (the default), all non-ASCII characters in the output are escaped with \uXXXX sequences, and the results are str instances consisting of ASCII characters only. If ensure_ascii is False, a result may be a unicode instance. This usually happens if the input contains unicode strings or the *encoding* parameter is used. If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to prevent an infinite recursion (which would cause an OverflowError). Otherwise, no such check takes place. If allow_nan is true, then NaN, Infinity, and -Infinity will be encoded as such. This behavior is not JSON specification compliant, but is consistent with most JavaScript based encoders and decoders. Otherwise, it will be a ValueError to encode such floats. If sort_keys is true, then the output of dictionaries will be sorted by key; this is useful for regression tests to ensure that JSON serializations can be compared on a day-to-day basis. If indent is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent level of 0 will only insert newlines. None is the most compact representation. Since the default item separator is ', ', the output might include trailing whitespace when indent is specified. You can use separators=(',', ': ') to avoid this. If specified, separators should be a (item_separator, key_separator) tuple. The default is (', ', ': '). To get the most compact JSON representation you should specify (',', ':') to eliminate whitespace. If specified, default is a function that gets called for objects that can't otherwise be serialized. It should return a JSON encodable version of the object or raise a ``TypeError``. If encoding is not None, then all input strings will be transformed into unicode using that encoding prior to JSON-encoding. The default is UTF-8. """ self.skipkeys = skipkeys self.ensure_ascii = ensure_ascii self.check_circular = check_circular self.allow_nan = allow_nan self.sort_keys = sort_keys self.indent = indent if separators is not None: self.item_separator, self.key_separator = separators if default is not None: self.default = default self.encoding = encoding def default(self, o): """Implement this method in a subclass such that it returns a serializable object for ``o``, or calls the base implementation (to raise a ``TypeError``). For example, to support arbitrary iterators, you could implement default like this:: def default(self, o): try: iterable = iter(o) except TypeError: pass else: return list(iterable) # Let the base class default method raise the TypeError return JSONEncoder.default(self, o) """ raise TypeError(repr(o) + " is not JSON serializable") def encode(self, o): """Return a JSON string representation of a Python data structure. >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) '{"foo": ["bar", "baz"]}' """ # This is for extremely simple cases and benchmarks. if isinstance(o, basestring): if isinstance(o, str): _encoding = self.encoding if (_encoding is not None and not (_encoding == 'utf-8')): o = o.decode(_encoding) if self.ensure_ascii: return encode_basestring_ascii(o) else: return encode_basestring(o) # This doesn't pass the iterator directly to ''.join() because the # exceptions aren't as detailed. The list call should be roughly # equivalent to the PySequence_Fast that ''.join() would do. chunks = self.iterencode(o, _one_shot=True) if not isinstance(chunks, (list, tuple)): chunks = list(chunks) return ''.join(chunks) def iterencode(self, o, _one_shot=False): """Encode the given object and yield each string representation as available. For example:: for chunk in JSONEncoder().iterencode(bigobject): mysocket.write(chunk) """ if self.check_circular: markers = {} else: markers = None if self.ensure_ascii: _encoder = encode_basestring_ascii else: _encoder = encode_basestring if self.encoding != 'utf-8': def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): if isinstance(o, str): o = o.decode(_encoding) return _orig_encoder(o) def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): # Check for specials. Note that this type of test is processor # and/or platform-specific, so do tests which don't depend on the # internals. if o != o: text = 'NaN' elif o == _inf: text = 'Infinity' elif o == _neginf: text = '-Infinity' else: return _repr(o) if not allow_nan: raise ValueError( "Out of range float values are not JSON compliant: " + repr(o)) return text if (_one_shot and c_make_encoder is not None and self.indent is None and not self.sort_keys): _iterencode = c_make_encoder( markers, self.default, _encoder, self.indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( markers, self.default, _encoder, self.indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot) return _iterencode(o, 0) def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, ## HACK: hand-optimized bytecode; turn globals into locals ValueError=ValueError, basestring=basestring, dict=dict, float=float, id=id, int=int, isinstance=isinstance, list=list, long=long, str=str, tuple=tuple, ): def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' return if markers is not None: markerid = id(lst) if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = lst buf = '[' if _indent is not None: _current_indent_level += 1 newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) separator = _item_separator + newline_indent buf += newline_indent else: newline_indent = None separator = _item_separator first = True for value in lst: if first: first = False else: buf = separator if isinstance(value, basestring): yield buf + _encoder(value) elif value is None: yield buf + 'null' elif value is True: yield buf + 'true' elif value is False: yield buf + 'false' elif isinstance(value, (int, long)): yield buf + str(value) elif isinstance(value, float): yield buf + _floatstr(value) else: yield buf if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) elif isinstance(value, dict): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) for chunk in chunks: yield chunk if newline_indent is not None: _current_indent_level -= 1 yield '\n' + (' ' * (_indent * _current_indent_level)) yield ']' if markers is not None: del markers[markerid] def _iterencode_dict(dct, _current_indent_level): if not dct: yield '{}' return if markers is not None: markerid = id(dct) if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = dct yield '{' if _indent is not None: _current_indent_level += 1 newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) item_separator = _item_separator + newline_indent yield newline_indent else: newline_indent = None item_separator = _item_separator first = True if _sort_keys: items = sorted(dct.items(), key=lambda kv: kv[0]) else: items = dct.iteritems() for key, value in items: if isinstance(key, basestring): pass # JavaScript is weakly typed for these, so it makes sense to # also allow them. Many encoders seem to do something like this. elif isinstance(key, float): key = _floatstr(key) elif key is True: key = 'true' elif key is False: key = 'false' elif key is None: key = 'null' elif isinstance(key, (int, long)): key = str(key) elif _skipkeys: continue else: raise TypeError("key " + repr(key) + " is not a string") if first: first = False else: yield item_separator yield _encoder(key) yield _key_separator if isinstance(value, basestring): yield _encoder(value) elif value is None: yield 'null' elif value is True: yield 'true' elif value is False: yield 'false' elif isinstance(value, (int, long)): yield str(value) elif isinstance(value, float): yield _floatstr(value) else: if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) elif isinstance(value, dict): chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) for chunk in chunks: yield chunk if newline_indent is not None: _current_indent_level -= 1 yield '\n' + (' ' * (_indent * _current_indent_level)) yield '}' if markers is not None: del markers[markerid] def _iterencode(o, _current_indent_level): if isinstance(o, basestring): yield _encoder(o) elif o is None: yield 'null' elif o is True: yield 'true' elif o is False: yield 'false' elif isinstance(o, (int, long)): yield str(o) elif isinstance(o, float): yield _floatstr(o) elif isinstance(o, (list, tuple)): for chunk in _iterencode_list(o, _current_indent_level): yield chunk elif isinstance(o, dict): for chunk in _iterencode_dict(o, _current_indent_level): yield chunk else: if markers is not None: markerid = id(o) if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = o o = _default(o) for chunk in _iterencode(o, _current_indent_level): yield chunk if markers is not None: del markers[markerid] return _iterencode ================================================ FILE: third_party/stdlib/json_scanner.py ================================================ """JSON token scanner """ import re # try: # from _json import make_scanner as c_make_scanner # except ImportError: # c_make_scanner = None c_make_scanner = None __all__ = ['make_scanner'] NUMBER_RE = re.compile( r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', (re.VERBOSE | re.MULTILINE | re.DOTALL)) def py_make_scanner(context): parse_object = context.parse_object parse_array = context.parse_array parse_string = context.parse_string match_number = NUMBER_RE.match encoding = context.encoding strict = context.strict parse_float = context.parse_float parse_int = context.parse_int parse_constant = context.parse_constant object_hook = context.object_hook object_pairs_hook = context.object_pairs_hook def _scan_once(string, idx): try: nextchar = string[idx] except IndexError: raise StopIteration if nextchar == '"': return parse_string(string, idx + 1, encoding, strict) elif nextchar == '{': return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook, object_pairs_hook) elif nextchar == '[': return parse_array((string, idx + 1), _scan_once) elif nextchar == 'n' and string[idx:idx + 4] == 'null': return None, idx + 4 elif nextchar == 't' and string[idx:idx + 4] == 'true': return True, idx + 4 elif nextchar == 'f' and string[idx:idx + 5] == 'false': return False, idx + 5 m = match_number(string, idx) if m is not None: integer, frac, exp = m.groups() if frac or exp: res = parse_float(integer + (frac or '') + (exp or '')) else: res = parse_int(integer) return res, m.end() elif nextchar == 'N' and string[idx:idx + 3] == 'NaN': return parse_constant('NaN'), idx + 3 elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity': return parse_constant('Infinity'), idx + 8 elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': return parse_constant('-Infinity'), idx + 9 else: raise StopIteration return _scan_once make_scanner = c_make_scanner or py_make_scanner ================================================ FILE: third_party/stdlib/keyword.py ================================================ #! /usr/bin/env python """Keywords (from "graminit.c") This file is automatically generated; please don't muck it up! To update the symbols in this file, 'cd' to the top directory of the python source tree after building the interpreter and run: ./python Lib/keyword.py """ __all__ = ["iskeyword", "kwlist"] kwlist = [ #--start keywords-- 'and', 'as', 'assert', 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or', 'pass', 'print', 'raise', 'return', 'try', 'while', 'with', 'yield', #--end keywords-- ] iskeyword = frozenset(kwlist).__contains__ def main(): import sys, re args = sys.argv[1:] iptfile = args and args[0] or "Python/graminit.c" if len(args) > 1: optfile = args[1] else: optfile = "Lib/keyword.py" # scan the source file for keywords fp = open(iptfile) strprog = re.compile('"([^"]+)"') lines = [] for line in fp: if '{1, "' in line: match = strprog.search(line) if match: lines.append(" '" + match.group(1) + "',\n") fp.close() lines.sort() # load the output skeleton from the target fp = open(optfile) format = fp.readlines() fp.close() # insert the lines of keywords try: start = format.index("#--start keywords--\n") + 1 end = format.index("#--end keywords--\n") format[start:end] = lines except ValueError: sys.stderr.write("target does not contain format markers\n") sys.exit(1) # write the output file fp = open(optfile, 'w') fp.write(''.join(format)) fp.close() if __name__ == "__main__": main() ================================================ FILE: third_party/stdlib/linecache.py ================================================ """Cache lines from files. This is intended to read lines from modules imported -- hence if a filename is not found, it will look down the module search path for a file by that name. """ import sys import os __all__ = ["getline", "clearcache", "checkcache"] def getline(filename, lineno, module_globals=None): lines = getlines(filename, module_globals) if 1 <= lineno <= len(lines): return lines[lineno-1] else: return '' # The cache cache = {} # The cache def clearcache(): """Clear the cache entirely.""" global cache cache = {} def getlines(filename, module_globals=None): """Get the lines for a file from the cache. Update the cache if it doesn't contain an entry for this file already.""" if filename in cache: return cache[filename][2] try: return updatecache(filename, module_globals) except MemoryError: clearcache() return [] def checkcache(filename=None): """Discard cache entries that are out of date. (This is not checked upon each call!)""" if filename is None: filenames = cache.keys() else: if filename in cache: filenames = [filename] else: return for filename in filenames: size, mtime, lines, fullname = cache[filename] if mtime is None: continue # no-op for files loaded via a __loader__ try: stat = os.stat(fullname) except os.error: del cache[filename] continue if size != stat.st_size or mtime != stat.st_mtime: del cache[filename] def updatecache(filename, module_globals=None): """Update a cache entry and return its list of lines. If something's wrong, print a message, discard the cache entry, and return an empty list.""" if filename in cache: del cache[filename] if not filename or (filename.startswith('<') and filename.endswith('>')): return [] fullname = filename try: stat = os.stat(fullname) except OSError: basename = filename # Try for a __loader__, if available if module_globals and '__loader__' in module_globals: name = module_globals.get('__name__') loader = module_globals['__loader__'] get_source = getattr(loader, 'get_source', None) if name and get_source: try: data = get_source(name) except (ImportError, IOError): pass else: if data is None: # No luck, the PEP302 loader cannot find the source # for this module. return [] cache[filename] = ( len(data), None, [line+'\n' for line in data.splitlines()], fullname ) return cache[filename][2] # Try looking through the module search path, which is only useful # when handling a relative filename. if os.path.isabs(filename): return [] for dirname in sys.path: # When using imputil, sys.path may contain things other than # strings; ignore them when it happens. try: fullname = os.path.join(dirname, basename) except (TypeError, AttributeError): # Not sufficiently string-like to do anything useful with. continue try: stat = os.stat(fullname) break except os.error: pass else: return [] try: with open(fullname, 'rU') as fp: lines = fp.readlines() except IOError: return [] if lines and not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname return lines ================================================ FILE: third_party/stdlib/md5.py ================================================ # $Id$ # # Copyright (C) 2005 Gregory P. Smith (greg@krypto.org) # Licensed to PSF under a Contributor Agreement. # import warnings # warnings.warn("the md5 module is deprecated; use hashlib instead", # DeprecationWarning, 2) # from hashlib import md5 import _md5 new = _md5.new md5 = _md5.new digest_size = 16 ================================================ FILE: third_party/stdlib/mimetools.py ================================================ """Various tools used by MIME-reading or MIME-writing programs.""" import os import sys import tempfile from warnings import filterwarnings, catch_warnings with catch_warnings(): if sys.py3kwarning: filterwarnings("ignore", ".*rfc822 has been removed", DeprecationWarning) import rfc822 from warnings import warnpy3k warnpy3k("in 3.x, mimetools has been removed in favor of the email package", stacklevel=2) __all__ = ["Message","choose_boundary","encode","decode","copyliteral", "copybinary"] class Message(rfc822.Message): """A derived class of rfc822.Message that knows about MIME headers and contains some hooks for decoding encoded and multipart messages.""" def __init__(self, fp, seekable = 1): rfc822.Message.__init__(self, fp, seekable) self.encodingheader = \ self.getheader('content-transfer-encoding') self.typeheader = \ self.getheader('content-type') self.parsetype() self.parseplist() def parsetype(self): str = self.typeheader if str is None: str = 'text/plain' if ';' in str: i = str.index(';') self.plisttext = str[i:] str = str[:i] else: self.plisttext = '' fields = str.split('/') for i in range(len(fields)): fields[i] = fields[i].strip().lower() self.type = '/'.join(fields) self.maintype = fields[0] self.subtype = '/'.join(fields[1:]) def parseplist(self): str = self.plisttext self.plist = [] while str[:1] == ';': str = str[1:] if ';' in str: # XXX Should parse quotes! end = str.index(';') else: end = len(str) f = str[:end] if '=' in f: i = f.index('=') f = f[:i].strip().lower() + \ '=' + f[i+1:].strip() self.plist.append(f.strip()) str = str[end:] def getplist(self): return self.plist def getparam(self, name): name = name.lower() + '=' n = len(name) for p in self.plist: if p[:n] == name: return rfc822.unquote(p[n:]) return None def getparamnames(self): result = [] for p in self.plist: i = p.find('=') if i >= 0: result.append(p[:i].lower()) return result def getencoding(self): if self.encodingheader is None: return '7bit' return self.encodingheader.lower() def gettype(self): return self.type def getmaintype(self): return self.maintype def getsubtype(self): return self.subtype # Utility functions # ----------------- #try: import thread #except ImportError: # import dummy_thread as thread _counter_lock = thread.allocate_lock() del thread _counter = 0 def _get_next_counter(): global _counter _counter_lock.acquire() _counter += 1 result = _counter _counter_lock.release() return result _prefix = None #def choose_boundary(): # """Return a string usable as a multipart boundary. # # The string chosen is unique within a single program run, and # incorporates the user id (if available), process id (if available), # and current time. So it's very unlikely the returned string appears # in message text, but there's no guarantee. # # The boundary contains dots so you have to quote it in the header.""" # # global _prefix # import time # if _prefix is None: # import socket # try: # hostid = socket.gethostbyname(socket.gethostname()) # except socket.gaierror: # hostid = '127.0.0.1' # try: # uid = repr(os.getuid()) # except AttributeError: # uid = '1' # try: # pid = repr(os.getpid()) # except AttributeError: # pid = '1' # _prefix = hostid + '.' + uid + '.' + pid # return "%s.%.3f.%d" % (_prefix, time.time(), _get_next_counter()) # Subroutines for decoding some common content-transfer-types def decode(input, output, encoding): """Decode common content-transfer-encodings (base64, quopri, uuencode).""" if encoding == 'base64': import base64 return base64.decode(input, output) if encoding == 'quoted-printable': import quopri return quopri.decode(input, output) if encoding in ('uuencode', 'x-uuencode', 'uue', 'x-uue'): import uu return uu.decode(input, output) if encoding in ('7bit', '8bit'): return output.write(input.read()) if encoding in decodetab: pipethrough(input, decodetab[encoding], output) else: raise ValueError, \ 'unknown Content-Transfer-Encoding: %s' % encoding def encode(input, output, encoding): """Encode common content-transfer-encodings (base64, quopri, uuencode).""" if encoding == 'base64': import base64 return base64.encode(input, output) if encoding == 'quoted-printable': import quopri return quopri.encode(input, output, 0) if encoding in ('uuencode', 'x-uuencode', 'uue', 'x-uue'): import uu return uu.encode(input, output) if encoding in ('7bit', '8bit'): return output.write(input.read()) if encoding in encodetab: pipethrough(input, encodetab[encoding], output) else: raise ValueError, \ 'unknown Content-Transfer-Encoding: %s' % encoding # The following is no longer used for standard encodings # XXX This requires that uudecode and mmencode are in $PATH uudecode_pipe = '''( TEMP=/tmp/@uu.$$ sed "s%^begin [0-7][0-7]* .*%begin 600 $TEMP%" | uudecode cat $TEMP rm $TEMP )''' decodetab = { 'uuencode': uudecode_pipe, 'x-uuencode': uudecode_pipe, 'uue': uudecode_pipe, 'x-uue': uudecode_pipe, 'quoted-printable': 'mmencode -u -q', 'base64': 'mmencode -u -b', } encodetab = { 'x-uuencode': 'uuencode tempfile', 'uuencode': 'uuencode tempfile', 'x-uue': 'uuencode tempfile', 'uue': 'uuencode tempfile', 'quoted-printable': 'mmencode -q', 'base64': 'mmencode -b', } def pipeto(input, command): pipe = os.popen(command, 'w') copyliteral(input, pipe) pipe.close() def pipethrough(input, command, output): (fd, tempname) = tempfile.mkstemp() temp = os.fdopen(fd, 'w') copyliteral(input, temp) temp.close() pipe = os.popen(command + ' <' + tempname, 'r') copybinary(pipe, output) pipe.close() os.unlink(tempname) def copyliteral(input, output): while 1: line = input.readline() if not line: break output.write(line) def copybinary(input, output): BUFSIZE = 8192 while 1: line = input.read(BUFSIZE) if not line: break output.write(line) ================================================ FILE: third_party/stdlib/mutex.py ================================================ """Mutual exclusion -- for use with module sched A mutex has two pieces of state -- a 'locked' bit and a queue. When the mutex is not locked, the queue is empty. Otherwise, the queue contains 0 or more (function, argument) pairs representing functions (or methods) waiting to acquire the lock. When the mutex is unlocked while the queue is not empty, the first queue entry is removed and its function(argument) pair called, implying it now has the lock. Of course, no multi-threading is implied -- hence the funny interface for lock, where a function is called once the lock is acquired. """ from warnings import warnpy3k warnpy3k("the mutex module has been removed in Python 3.0", stacklevel=2) del warnpy3k from collections import deque class mutex(object): def __init__(self): """Create a new mutex -- initially unlocked.""" self.locked = False self.queue = deque() def test(self): """Test the locked bit of the mutex.""" return self.locked def testandset(self): """Atomic test-and-set -- grab the lock if it is not set, return True if it succeeded.""" if not self.locked: self.locked = True return True else: return False def lock(self, function, argument): """Lock a mutex, call the function with supplied argument when it is acquired. If the mutex is already locked, place function and argument in the queue.""" if self.testandset(): function(argument) else: self.queue.append((function, argument)) def unlock(self): """Unlock a mutex. If the queue is not empty, call the next function with its argument.""" if self.queue: function, argument = self.queue.popleft() function(argument) else: self.locked = False ================================================ FILE: third_party/stdlib/optparse.py ================================================ """A powerful, extensible, and easy-to-use option parser. By Greg Ward Originally distributed as Optik. For support, use the optik-users@lists.sourceforge.net mailing list (http://lists.sourceforge.net/lists/listinfo/optik-users). Simple usage example: from optparse import OptionParser parser = OptionParser() parser.add_option("-f", "--file", dest="filename", help="write report to FILE", metavar="FILE") parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True, help="don't print status messages to stdout") (options, args) = parser.parse_args() """ __version__ = "1.5.3" __all__ = ['Option', 'make_option', 'SUPPRESS_HELP', 'SUPPRESS_USAGE', 'Values', 'OptionContainer', 'OptionGroup', 'OptionParser', 'HelpFormatter', 'IndentedHelpFormatter', 'TitledHelpFormatter', 'OptParseError', 'OptionError', 'OptionConflictError', 'OptionValueError', 'BadOptionError'] __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. Copyright (c) 2002-2006 Python Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ import sys, os import types import textwrap def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) def setattr(self, attr, value): self.__dict__[attr] = value INDENT_CHAR = " " # This file was generated from: # Id: option_parser.py 527 2006-07-23 15:21:30Z greg # Id: option.py 522 2006-06-11 16:22:03Z gward # Id: help.py 527 2006-07-23 15:21:30Z greg # Id: errors.py 509 2006-04-20 00:58:24Z gward # try: # from gettext import gettext # except ImportError: # def gettext(message): # return message def gettext(message): return message _ = gettext class OptParseError (Exception): def __init__(self, msg): self.msg = msg def __str__(self): return self.msg class OptionError (OptParseError): """ Raised if an Option instance is created with invalid or inconsistent arguments. """ def __init__(self, msg, option): self.msg = msg self.option_id = str(option) def __str__(self): if self.option_id: return "option %s: %s" % (self.option_id, self.msg) else: return self.msg class OptionConflictError (OptionError): """ Raised if conflicting options are added to an OptionParser. """ class OptionValueError (OptParseError): """ Raised if an invalid option value is encountered on the command line. """ class BadOptionError (OptParseError): """ Raised if an invalid option is seen on the command line. """ def __init__(self, opt_str): self.opt_str = opt_str def __str__(self): return _("no such option: %s") % self.opt_str class AmbiguousOptionError (BadOptionError): """ Raised if an ambiguous option is seen on the command line. """ def __init__(self, opt_str, possibilities): BadOptionError.__init__(self, opt_str) self.possibilities = possibilities def __str__(self): return (_("ambiguous option: %s (%s?)") % (self.opt_str, ", ".join(self.possibilities))) class HelpFormatter(object): """ Abstract base class for formatting option help. OptionParser instances should use one of the HelpFormatter subclasses for formatting help; by default IndentedHelpFormatter is used. Instance attributes: parser : OptionParser the controlling OptionParser instance indent_increment : int the number of columns to indent per nesting level max_help_position : int the maximum starting column for option help text help_position : int the calculated starting column for option help text; initially the same as the maximum width : int total number of columns for output (pass None to constructor for this value to be taken from the $COLUMNS environment variable) level : int current indentation level current_indent : int current indentation level (in columns) help_width : int number of columns available for option help text (calculated) default_tag : str text to replace with each option's default value, "%default" by default. Set to false value to disable default value expansion. option_strings : { Option : str } maps Option instances to the snippet of help text explaining the syntax of that option, e.g. "-h, --help" or "-fFILE, --file=FILE" _short_opt_fmt : str format string controlling how short options with values are printed in help text. Must be either "%s%s" ("-fFILE") or "%s %s" ("-f FILE"), because those are the two syntaxes that Optik supports. _long_opt_fmt : str similar but for long options; must be either "%s %s" ("--file FILE") or "%s=%s" ("--file=FILE"). """ NO_DEFAULT_VALUE = "none" def __init__(self, indent_increment, max_help_position, width, short_first): self.parser = None self.indent_increment = indent_increment if width is None: try: width = int(os.environ['COLUMNS']) except (KeyError, ValueError): width = 80 width -= 2 self.width = width self.help_position = self.max_help_position = \ min(max_help_position, max(width - 20, indent_increment * 2)) self.current_indent = 0 self.level = 0 self.help_width = None # computed later self.short_first = short_first self.default_tag = "%default" self.option_strings = {} self._short_opt_fmt = "%s %s" self._long_opt_fmt = "%s=%s" def set_parser(self, parser): self.parser = parser def set_short_opt_delimiter(self, delim): if delim not in ("", " "): raise ValueError( "invalid metavar delimiter for short options: %r" % delim) self._short_opt_fmt = "%s" + delim + "%s" def set_long_opt_delimiter(self, delim): if delim not in ("=", " "): raise ValueError( "invalid metavar delimiter for long options: %r" % delim) self._long_opt_fmt = "%s" + delim + "%s" def indent(self): self.current_indent += self.indent_increment self.level += 1 def dedent(self): self.current_indent -= self.indent_increment assert self.current_indent >= 0, "Indent decreased below 0." self.level -= 1 def format_usage(self, usage): raise NotImplementedError, "subclasses must implement" def format_heading(self, heading): raise NotImplementedError, "subclasses must implement" def _format_text(self, text): """ Format a paragraph of free-form text for inclusion in the help output at the current indentation level. """ text_width = max(self.width - self.current_indent, 11) indent = " "*self.current_indent return textwrap.fill(text, text_width, initial_indent=indent, subsequent_indent=indent) def format_description(self, description): if description: return self._format_text(description) + "\n" else: return "" def format_epilog(self, epilog): if epilog: return "\n" + self._format_text(epilog) + "\n" else: return "" def expand_default(self, option): if self.parser is None or not self.default_tag: return option.help default_value = self.parser.defaults.get(option.dest) if default_value is NO_DEFAULT or default_value is None: default_value = self.NO_DEFAULT_VALUE # return option.help.replace(self.default_tag, str(default_value)) return str(default_value).join(option.help.split(self.default_tag)) def format_option(self, option): # The help for each option consists of two parts: # * the opt strings and metavars # eg. ("-x", or "-fFILENAME, --file=FILENAME") # * the user-supplied help string # eg. ("turn on expert mode", "read data from FILENAME") # # If possible, we write both of these on the same line: # -x turn on expert mode # # But if the opt string list is too long, we put the help # string on a second line, indented to the same column it would # start in if it fit on the first line. # -fFILENAME, --file=FILENAME # read data from FILENAME result = [] opts = self.option_strings[option] opt_width = self.help_position - self.current_indent - 2 if len(opts) > opt_width: # opts = "%*s%s\n" % (self.current_indent, "", opts) opts = "%s%s\n" % (self.current_indent * INDENT_CHAR + "", opts) indent_first = self.help_position else: # start help on same line as opts # opts = "%*s%-*s " % (self.current_indent, "", opt_width, opts) opts = "%s%s " % (self.current_indent * INDENT_CHAR, (opts + opt_width * INDENT_CHAR)[:opt_width]) indent_first = 0 result.append(opts) if option.help: help_text = self.expand_default(option) help_lines = textwrap.wrap(help_text, self.help_width) # result.append("%*s%s\n" % (indent_first, "", help_lines[0])) result.append("%s%s\n" % (indent_first * INDENT_CHAR + "", help_lines[0])) # result.extend(["%*s%s\n" % (self.help_position, "", line) # result.extend(["%s%s\n" % (self.help_position * INDENT_CHAR + "", line) # for line in help_lines[1:]]) result += (["%s%s\n" % (self.help_position * INDENT_CHAR + "", line) for line in help_lines[1:]]) elif opts[-1] != "\n": result.append("\n") return "".join(result) def store_option_strings(self, parser): self.indent() max_len = 0 for opt in parser.option_list: strings = self.format_option_strings(opt) self.option_strings[opt] = strings max_len = max(max_len, len(strings) + self.current_indent) self.indent() for group in parser.option_groups: for opt in group.option_list: strings = self.format_option_strings(opt) self.option_strings[opt] = strings max_len = max(max_len, len(strings) + self.current_indent) self.dedent() self.dedent() self.help_position = min(max_len + 2, self.max_help_position) self.help_width = max(self.width - self.help_position, 11) def format_option_strings(self, option): """Return a comma-separated list of option strings & metavariables.""" if option.takes_value(): metavar = option.metavar or option.dest.upper() short_opts = [self._short_opt_fmt % (sopt, metavar) for sopt in option._short_opts] long_opts = [self._long_opt_fmt % (lopt, metavar) for lopt in option._long_opts] else: short_opts = option._short_opts long_opts = option._long_opts if self.short_first: opts = short_opts + long_opts else: opts = long_opts + short_opts return ", ".join(opts) class IndentedHelpFormatter (HelpFormatter): """Format help with indented section bodies. """ def __init__(self, indent_increment=2, max_help_position=24, width=None, short_first=1): HelpFormatter.__init__( self, indent_increment, max_help_position, width, short_first) def format_usage(self, usage): return _("Usage: %s\n") % usage def format_heading(self, heading): # return "%*s%s:\n" % (self.current_indent, "", heading) return "%s%s:\n" % (self.current_indent * INDENT_CHAR + "", heading) class TitledHelpFormatter (HelpFormatter): """Format help with underlined section headers. """ def __init__(self, indent_increment=0, max_help_position=24, width=None, short_first=0): HelpFormatter.__init__ ( self, indent_increment, max_help_position, width, short_first) def format_usage(self, usage): return "%s %s\n" % (self.format_heading(_("Usage")), usage) def format_heading(self, heading): return "%s\n%s\n" % (heading, "=-"[self.level] * len(heading)) def _parse_num(val, type): if val[:2].lower() == "0x": # hexadecimal radix = 16 elif val[:2].lower() == "0b": # binary radix = 2 val = val[2:] or "0" # have to remove "0b" prefix elif val[:1] == "0": # octal radix = 8 else: # decimal radix = 10 return type(val, radix) def _parse_int(val): return _parse_num(val, int) def _parse_long(val): return _parse_num(val, long) _builtin_cvt = { "int" : (_parse_int, _("integer")), "long" : (_parse_long, _("long integer")), "float" : (float, _("floating-point")) } # "complex" : (complex, _("complex")) } def check_builtin(option, opt, value): (cvt, what) = _builtin_cvt[option.type] try: return cvt(value) except ValueError: raise OptionValueError( _("option %s: invalid %s value: %r") % (opt, what, value)) def check_choice(option, opt, value): if value in option.choices: return value else: choices = ", ".join(map(repr, option.choices)) raise OptionValueError( _("option %s: invalid choice: %r (choose from %s)") % (opt, value, choices)) # Not supplying a default is different from a default of None, # so we need an explicit "not supplied" value. NO_DEFAULT = ("NO", "DEFAULT") class Option(object): """ Instance attributes: _short_opts : [string] _long_opts : [string] action : string type : string dest : string default : any nargs : int const : any choices : [string] callback : function callback_args : (any*) callback_kwargs : { string : any } help : string metavar : string """ # The list of instance attributes that may be set through # keyword args to the constructor. ATTRS = ['action', 'type', 'dest', 'default', 'nargs', 'const', 'choices', 'callback', 'callback_args', 'callback_kwargs', 'help', 'metavar'] # The set of actions allowed by option parsers. Explicitly listed # here so the constructor can validate its arguments. ACTIONS = ("store", "store_const", "store_true", "store_false", "append", "append_const", "count", "callback", "help", "version") # The set of actions that involve storing a value somewhere; # also listed just for constructor argument validation. (If # the action is one of these, there must be a destination.) STORE_ACTIONS = ("store", "store_const", "store_true", "store_false", "append", "append_const", "count") # The set of actions for which it makes sense to supply a value # type, ie. which may consume an argument from the command line. TYPED_ACTIONS = ("store", "append", "callback") # The set of actions which *require* a value type, ie. that # always consume an argument from the command line. ALWAYS_TYPED_ACTIONS = ("store", "append") # The set of actions which take a 'const' attribute. CONST_ACTIONS = ("store_const", "append_const") # The set of known types for option parsers. Again, listed here for # constructor argument validation. TYPES = ("string", "int", "long", "float", "choice") #, "complex" # Dictionary of argument checking functions, which convert and # validate option arguments according to the option type. # # Signature of checking functions is: # check(option : Option, opt : string, value : string) -> any # where # option is the Option instance calling the checker # opt is the actual option seen on the command-line # (eg. "-a", "--file") # value is the option argument seen on the command-line # # The return value should be in the appropriate Python type # for option.type -- eg. an integer if option.type == "int". # # If no checker is defined for a type, arguments will be # unchecked and remain strings. TYPE_CHECKER = { "int" : check_builtin, "long" : check_builtin, "float" : check_builtin, # "complex": check_builtin, "choice" : check_choice, } # CHECK_METHODS is a list of unbound method objects; they are called # by the constructor, in order, after all attributes are # initialized. The list is created and filled in later, after all # the methods are actually defined. (I just put it here because I # like to define and document all class attributes in the same # place.) Subclasses that add another _check_*() method should # define their own CHECK_METHODS list that adds their check method # to those from this class. CHECK_METHODS = None # -- Constructor/initialization methods ---------------------------- def __init__(self, *opts, **attrs): # Set _short_opts, _long_opts attrs from 'opts' tuple. # Have to be set now, in case no option strings are supplied. self._short_opts = [] self._long_opts = [] opts = self._check_opt_strings(opts) self._set_opt_strings(opts) # Set all other attrs (action, type, etc.) from 'attrs' dict self._set_attrs(attrs) # Check all the attributes we just set. There are lots of # complicated interdependencies, but luckily they can be farmed # out to the _check_*() methods listed in CHECK_METHODS -- which # could be handy for subclasses! The one thing these all share # is that they raise OptionError if they discover a problem. for checker in self.CHECK_METHODS: checker(self) def _check_opt_strings(self, opts): # Filter out None because early versions of Optik had exactly # one short option and one long option, either of which # could be None. # opts = filter(None, opts) opts = [x for x in opts if x is not None] if not opts: raise TypeError("at least one option string must be supplied") return opts def _set_opt_strings(self, opts): for opt in opts: if len(opt) < 2: raise OptionError( "invalid option string %r: " "must be at least two characters long" % opt, self) elif len(opt) == 2: if not (opt[0] == "-" and opt[1] != "-"): raise OptionError( "invalid short option string %r: " "must be of the form -x, (x any non-dash char)" % opt, self) self._short_opts.append(opt) else: if not (opt[0:2] == "--" and opt[2] != "-"): raise OptionError( "invalid long option string %r: " "must start with --, followed by non-dash" % opt, self) self._long_opts.append(opt) def _set_attrs(self, attrs): for attr in self.ATTRS: if attr in attrs: setattr(self, attr, attrs[attr]) del attrs[attr] else: if attr == 'default': setattr(self, attr, NO_DEFAULT) else: setattr(self, attr, None) if attrs: attrs = attrs.keys() attrs.sort() raise OptionError( "invalid keyword arguments: %s" % ", ".join(attrs), self) # -- Constructor validation methods -------------------------------- def _check_action(self): if self.action is None: self.action = "store" elif self.action not in self.ACTIONS: raise OptionError("invalid action: %r" % self.action, self) def _check_type(self): if self.type is None: if self.action in self.ALWAYS_TYPED_ACTIONS: if self.choices is not None: # The "choices" attribute implies "choice" type. self.type = "choice" else: # No type given? "string" is the most sensible default. self.type = "string" else: # Allow type objects or builtin type conversion functions # (int, str, etc.) as an alternative to their names. (The # complicated check of __builtin__ is only necessary for # Python 2.1 and earlier, and is short-circuited by the # first check on modern Pythons.) import __builtin__ if ( type(self.type) is types.TypeType or (hasattr(self.type, "__name__") and getattr(__builtin__, self.type.__name__, None) is self.type) ): self.type = self.type.__name__ if self.type == "str": self.type = "string" if self.type not in self.TYPES: raise OptionError("invalid option type: %r" % self.type, self) if self.action not in self.TYPED_ACTIONS: raise OptionError( "must not supply a type for action %r" % self.action, self) def _check_choice(self): if self.type == "choice": if self.choices is None: raise OptionError( "must supply a list of choices for type 'choice'", self) elif type(self.choices) not in (types.TupleType, types.ListType): raise OptionError( "choices must be a list of strings ('%s' supplied)" % str(type(self.choices)).split("'")[1], self) elif self.choices is not None: raise OptionError( "must not supply choices for type %r" % self.type, self) def _check_dest(self): # No destination given, and we need one for this action. The # self.type check is for callbacks that take a value. takes_value = (self.action in self.STORE_ACTIONS or self.type is not None) if self.dest is None and takes_value: # Glean a destination from the first long option string, # or from the first short option string if no long options. if self._long_opts: # eg. "--foo-bar" -> "foo_bar" # self.dest = self._long_opts[0][2:].replace('-', '_') self.dest = '_'.join(self._long_opts[0][2:].split('-')) else: self.dest = self._short_opts[0][1] def _check_const(self): if self.action not in self.CONST_ACTIONS and self.const is not None: raise OptionError( "'const' must not be supplied for action %r" % self.action, self) def _check_nargs(self): if self.action in self.TYPED_ACTIONS: if self.nargs is None: self.nargs = 1 elif self.nargs is not None: raise OptionError( "'nargs' must not be supplied for action %r" % self.action, self) def _check_callback(self): if self.action == "callback": if not hasattr(self.callback, '__call__'): raise OptionError( "callback not callable: %r" % self.callback, self) if (self.callback_args is not None and type(self.callback_args) is not types.TupleType): raise OptionError( "callback_args, if supplied, must be a tuple: not %r" % self.callback_args, self) if (self.callback_kwargs is not None and type(self.callback_kwargs) is not types.DictType): raise OptionError( "callback_kwargs, if supplied, must be a dict: not %r" % self.callback_kwargs, self) else: if self.callback is not None: raise OptionError( "callback supplied (%r) for non-callback option" % self.callback, self) if self.callback_args is not None: raise OptionError( "callback_args supplied for non-callback option", self) if self.callback_kwargs is not None: raise OptionError( "callback_kwargs supplied for non-callback option", self) CHECK_METHODS = [_check_action, _check_type, _check_choice, _check_dest, _check_const, _check_nargs, _check_callback] # -- Miscellaneous methods ----------------------------------------- def __str__(self): return "/".join(self._short_opts + self._long_opts) __repr__ = _repr def takes_value(self): return self.type is not None def get_opt_string(self): if self._long_opts: return self._long_opts[0] else: return self._short_opts[0] # -- Processing methods -------------------------------------------- def check_value(self, opt, value): checker = self.TYPE_CHECKER.get(self.type) if checker is None: return value else: return checker(self, opt, value) def convert_value(self, opt, value): if value is not None: if self.nargs == 1: return self.check_value(opt, value) else: return tuple([self.check_value(opt, v) for v in value]) def process(self, opt, value, values, parser): # First, convert the value(s) to the right type. Howl if any # value(s) are bogus. value = self.convert_value(opt, value) # And then take whatever action is expected of us. # This is a separate method to make life easier for # subclasses to add new actions. return self.take_action( self.action, self.dest, opt, value, values, parser) def take_action(self, action, dest, opt, value, values, parser): if action == "store": setattr(values, dest, value) elif action == "store_const": setattr(values, dest, self.const) elif action == "store_true": setattr(values, dest, True) elif action == "store_false": setattr(values, dest, False) elif action == "append": values.ensure_value(dest, []).append(value) elif action == "append_const": values.ensure_value(dest, []).append(self.const) elif action == "count": setattr(values, dest, values.ensure_value(dest, 0) + 1) elif action == "callback": args = self.callback_args or () kwargs = self.callback_kwargs or {} self.callback(self, opt, value, parser, *args, **kwargs) elif action == "help": parser.print_help() parser.exit() elif action == "version": parser.print_version() parser.exit() else: raise ValueError("unknown action %r" % self.action) return 1 # class Option SUPPRESS_HELP = "SUPPRESS"+"HELP" SUPPRESS_USAGE = "SUPPRESS"+"USAGE" try: basestring except NameError: def isbasestring(x): return isinstance(x, (types.StringType, types.UnicodeType)) else: def isbasestring(x): return isinstance(x, basestring) class Values(object): def __init__(self, defaults=None): if defaults: for (attr, val) in defaults.items(): setattr(self, attr, val) def __str__(self): return str(self.__dict__) __repr__ = _repr def __cmp__(self, other): if isinstance(other, Values): return cmp(self.__dict__, other.__dict__) elif isinstance(other, types.DictType): return cmp(self.__dict__, other) else: return -1 def _update_careful(self, dict): """ Update the option values from an arbitrary dictionary, but only use keys from dict that already have a corresponding attribute in self. Any keys in dict without a corresponding attribute are silently ignored. """ for attr in dir(self): if attr in dict: dval = dict[attr] if dval is not None: setattr(self, attr, dval) def _update_loose(self, dict): """ Update the option values from an arbitrary dictionary, using all keys from the dictionary regardless of whether they have a corresponding attribute in self or not. """ self.__dict__.update(dict) def _update(self, dict, mode): if mode == "careful": self._update_careful(dict) elif mode == "loose": self._update_loose(dict) else: raise ValueError, "invalid update mode: %r" % mode def read_module(self, modname, mode="careful"): __import__(modname) mod = sys.modules[modname] self._update(vars(mod), mode) def read_file(self, filename, mode="careful"): vars = {} execfile(filename, vars) self._update(vars, mode) def ensure_value(self, attr, value): if not hasattr(self, attr) or getattr(self, attr) is None: setattr(self, attr, value) return getattr(self, attr) class OptionContainer(object): """ Abstract base class. Class attributes: standard_option_list : [Option] list of standard options that will be accepted by all instances of this parser class (intended to be overridden by subclasses). Instance attributes: option_list : [Option] the list of Option objects contained by this OptionContainer _short_opt : { string : Option } dictionary mapping short option strings, eg. "-f" or "-X", to the Option instances that implement them. If an Option has multiple short option strings, it will appear in this dictionary multiple times. [1] _long_opt : { string : Option } dictionary mapping long option strings, eg. "--file" or "--exclude", to the Option instances that implement them. Again, a given Option can occur multiple times in this dictionary. [1] defaults : { string : any } dictionary mapping option destination names to default values for each destination [1] [1] These mappings are common to (shared by) all components of the controlling OptionParser, where they are initially created. """ def __init__(self, option_class, conflict_handler, description): # Initialize the option list and related data structures. # This method must be provided by subclasses, and it must # initialize at least the following instance attributes: # option_list, _short_opt, _long_opt, defaults. self._create_option_list() self.option_class = option_class self.set_conflict_handler(conflict_handler) self.set_description(description) def _create_option_mappings(self): # For use by OptionParser constructor -- create the master # option mappings used by this OptionParser and all # OptionGroups that it owns. self._short_opt = {} # single letter -> Option instance self._long_opt = {} # long option -> Option instance self.defaults = {} # maps option dest -> default value def _share_option_mappings(self, parser): # For use by OptionGroup constructor -- use shared option # mappings from the OptionParser that owns this OptionGroup. self._short_opt = parser._short_opt self._long_opt = parser._long_opt self.defaults = parser.defaults def set_conflict_handler(self, handler): if handler not in ("error", "resolve"): raise ValueError, "invalid conflict_resolution value %r" % handler self.conflict_handler = handler def set_description(self, description): self.description = description def get_description(self): return self.description def destroy(self): """see OptionParser.destroy().""" del self._short_opt del self._long_opt del self.defaults # -- Option-adding methods ----------------------------------------- def _check_conflict(self, option): conflict_opts = [] for opt in option._short_opts: if opt in self._short_opt: conflict_opts.append((opt, self._short_opt[opt])) for opt in option._long_opts: if opt in self._long_opt: conflict_opts.append((opt, self._long_opt[opt])) if conflict_opts: handler = self.conflict_handler if handler == "error": raise OptionConflictError( "conflicting option string(s): %s" % ", ".join([co[0] for co in conflict_opts]), option) elif handler == "resolve": for (opt, c_option) in conflict_opts: if opt.startswith("--"): c_option._long_opts.remove(opt) del self._long_opt[opt] else: c_option._short_opts.remove(opt) del self._short_opt[opt] if not (c_option._short_opts or c_option._long_opts): c_option.container.option_list.remove(c_option) def add_option(self, *args, **kwargs): """add_option(Option) add_option(opt_str, ..., kwarg=val, ...) """ if type(args[0]) in types.StringTypes: option = self.option_class(*args, **kwargs) elif len(args) == 1 and not kwargs: option = args[0] if not isinstance(option, Option): raise TypeError, "not an Option instance: %r" % option else: raise TypeError, "invalid arguments" self._check_conflict(option) self.option_list.append(option) option.container = self for opt in option._short_opts: self._short_opt[opt] = option for opt in option._long_opts: self._long_opt[opt] = option if option.dest is not None: # option has a dest, we need a default if option.default is not NO_DEFAULT: self.defaults[option.dest] = option.default elif option.dest not in self.defaults: self.defaults[option.dest] = None return option def add_options(self, option_list): for option in option_list: self.add_option(option) # -- Option query/removal methods ---------------------------------- def get_option(self, opt_str): return (self._short_opt.get(opt_str) or self._long_opt.get(opt_str)) def has_option(self, opt_str): return (opt_str in self._short_opt or opt_str in self._long_opt) def remove_option(self, opt_str): option = self._short_opt.get(opt_str) if option is None: option = self._long_opt.get(opt_str) if option is None: raise ValueError("no such option %r" % opt_str) for opt in option._short_opts: del self._short_opt[opt] for opt in option._long_opts: del self._long_opt[opt] option.container.option_list.remove(option) # -- Help-formatting methods --------------------------------------- def format_option_help(self, formatter): if not self.option_list: return "" result = [] for option in self.option_list: if not option.help is SUPPRESS_HELP: result.append(formatter.format_option(option)) return "".join(result) def format_description(self, formatter): return formatter.format_description(self.get_description()) def format_help(self, formatter): result = [] if self.description: result.append(self.format_description(formatter)) if self.option_list: result.append(self.format_option_help(formatter)) return "\n".join(result) class OptionGroup (OptionContainer): def __init__(self, parser, title, description=None): self.parser = parser OptionContainer.__init__( self, parser.option_class, parser.conflict_handler, description) self.title = title def _create_option_list(self): self.option_list = [] self._share_option_mappings(self.parser) def set_title(self, title): self.title = title def destroy(self): """see OptionParser.destroy().""" OptionContainer.destroy(self) del self.option_list # -- Help-formatting methods --------------------------------------- def format_help(self, formatter): result = formatter.format_heading(self.title) formatter.indent() result += OptionContainer.format_help(self, formatter) formatter.dedent() return result class OptionParser (OptionContainer): """ Class attributes: standard_option_list : [Option] list of standard options that will be accepted by all instances of this parser class (intended to be overridden by subclasses). Instance attributes: usage : string a usage string for your program. Before it is displayed to the user, "%prog" will be expanded to the name of your program (self.prog or os.path.basename(sys.argv[0])). prog : string the name of the current program (to override os.path.basename(sys.argv[0])). description : string A paragraph of text giving a brief overview of your program. optparse reformats this paragraph to fit the current terminal width and prints it when the user requests help (after usage, but before the list of options). epilog : string paragraph of help text to print after option help option_groups : [OptionGroup] list of option groups in this parser (option groups are irrelevant for parsing the command-line, but very useful for generating help) allow_interspersed_args : bool = true if true, positional arguments may be interspersed with options. Assuming -a and -b each take a single argument, the command-line -ablah foo bar -bboo baz will be interpreted the same as -ablah -bboo -- foo bar baz If this flag were false, that command line would be interpreted as -ablah -- foo bar -bboo baz -- ie. we stop processing options as soon as we see the first non-option argument. (This is the tradition followed by Python's getopt module, Perl's Getopt::Std, and other argument- parsing libraries, but it is generally annoying to users.) process_default_values : bool = true if true, option default values are processed similarly to option values from the command line: that is, they are passed to the type-checking function for the option's type (as long as the default value is a string). (This really only matters if you have defined custom types; see SF bug #955889.) Set it to false to restore the behaviour of Optik 1.4.1 and earlier. rargs : [string] the argument list currently being parsed. Only set when parse_args() is active, and continually trimmed down as we consume arguments. Mainly there for the benefit of callback options. largs : [string] the list of leftover arguments that we have skipped while parsing options. If allow_interspersed_args is false, this list is always empty. values : Values the set of option values currently being accumulated. Only set when parse_args() is active. Also mainly for callbacks. Because of the 'rargs', 'largs', and 'values' attributes, OptionParser is not thread-safe. If, for some perverse reason, you need to parse command-line arguments simultaneously in different threads, use different OptionParser instances. """ standard_option_list = [] def __init__(self, usage=None, option_list=None, option_class=Option, version=None, conflict_handler="error", description=None, formatter=None, add_help_option=True, prog=None, epilog=None): OptionContainer.__init__( self, option_class, conflict_handler, description) self.set_usage(usage) self.prog = prog self.version = version self.allow_interspersed_args = True self.process_default_values = True if formatter is None: formatter = IndentedHelpFormatter() self.formatter = formatter self.formatter.set_parser(self) self.epilog = epilog # Populate the option list; initial sources are the # standard_option_list class attribute, the 'option_list' # argument, and (if applicable) the _add_version_option() and # _add_help_option() methods. self._populate_option_list(option_list, add_help=add_help_option) self._init_parsing_state() def destroy(self): """ Declare that you are done with this OptionParser. This cleans up reference cycles so the OptionParser (and all objects referenced by it) can be garbage-collected promptly. After calling destroy(), the OptionParser is unusable. """ OptionContainer.destroy(self) for group in self.option_groups: group.destroy() del self.option_list del self.option_groups del self.formatter # -- Private methods ----------------------------------------------- # (used by our or OptionContainer's constructor) def _create_option_list(self): self.option_list = [] self.option_groups = [] self._create_option_mappings() def _add_help_option(self): self.add_option("-h", "--help", action="help", help=_("show this help message and exit")) def _add_version_option(self): self.add_option("--version", action="version", help=_("show program's version number and exit")) def _populate_option_list(self, option_list, add_help=True): if self.standard_option_list: self.add_options(self.standard_option_list) if option_list: self.add_options(option_list) if self.version: self._add_version_option() if add_help: self._add_help_option() def _init_parsing_state(self): # These are set in parse_args() for the convenience of callbacks. self.rargs = None self.largs = None self.values = None # -- Simple modifier methods --------------------------------------- def set_usage(self, usage): if usage is None: self.usage = _("%prog [options]") elif usage is SUPPRESS_USAGE: self.usage = None # For backwards compatibility with Optik 1.3 and earlier. elif usage.lower().startswith("usage: "): self.usage = usage[7:] else: self.usage = usage def enable_interspersed_args(self): """Set parsing to not stop on the first non-option, allowing interspersing switches with command arguments. This is the default behavior. See also disable_interspersed_args() and the class documentation description of the attribute allow_interspersed_args.""" self.allow_interspersed_args = True def disable_interspersed_args(self): """Set parsing to stop on the first non-option. Use this if you have a command processor which runs another command that has options of its own and you want to make sure these options don't get confused. """ self.allow_interspersed_args = False def set_process_default_values(self, process): self.process_default_values = process def set_default(self, dest, value): self.defaults[dest] = value def set_defaults(self, **kwargs): self.defaults.update(kwargs) def _get_all_options(self): options = self.option_list[:] for group in self.option_groups: # options.extend(group.option_list) options += (group.option_list) return options def get_default_values(self): if not self.process_default_values: # Old, pre-Optik 1.5 behaviour. return Values(self.defaults) # defaults = self.defaults.copy() defaults = dict(self.defaults) for option in self._get_all_options(): default = defaults.get(option.dest) if isbasestring(default): opt_str = option.get_opt_string() defaults[option.dest] = option.check_value(opt_str, default) return Values(defaults) # -- OptionGroup methods ------------------------------------------- def add_option_group(self, *args, **kwargs): # XXX lots of overlap with OptionContainer.add_option() if type(args[0]) is types.StringType: group = OptionGroup(self, *args, **kwargs) elif len(args) == 1 and not kwargs: group = args[0] if not isinstance(group, OptionGroup): raise TypeError, "not an OptionGroup instance: %r" % group if group.parser is not self: raise ValueError, "invalid OptionGroup (wrong parser)" else: raise TypeError, "invalid arguments" self.option_groups.append(group) return group def get_option_group(self, opt_str): option = (self._short_opt.get(opt_str) or self._long_opt.get(opt_str)) if option and option.container is not self: return option.container return None # -- Option-parsing methods ---------------------------------------- def _get_args(self, args): if args is None: return sys.argv[1:] else: return args[:] # don't modify caller's list def parse_args(self, args=None, values=None): """ parse_args(args : [string] = sys.argv[1:], values : Values = None) -> (values : Values, args : [string]) Parse the command-line options found in 'args' (default: sys.argv[1:]). Any errors result in a call to 'error()', which by default prints the usage message to stderr and calls sys.exit() with an error message. On success returns a pair (values, args) where 'values' is a Values instance (with all your option values) and 'args' is the list of arguments left over after parsing options. """ rargs = self._get_args(args) if values is None: values = self.get_default_values() # Store the halves of the argument list as attributes for the # convenience of callbacks: # rargs # the rest of the command-line (the "r" stands for # "remaining" or "right-hand") # largs # the leftover arguments -- ie. what's left after removing # options and their arguments (the "l" stands for "leftover" # or "left-hand") self.rargs = rargs self.largs = largs = [] self.values = values try: stop = self._process_args(largs, rargs, values) except (BadOptionError, OptionValueError), err: self.error(str(err)) args = largs + rargs return self.check_values(values, args) def check_values(self, values, args): """ check_values(values : Values, args : [string]) -> (values : Values, args : [string]) Check that the supplied option values and leftover arguments are valid. Returns the option values and leftover arguments (possibly adjusted, possibly completely new -- whatever you like). Default implementation just returns the passed-in values; subclasses may override as desired. """ return (values, args) def _process_args(self, largs, rargs, values): """_process_args(largs : [string], rargs : [string], values : Values) Process command-line arguments and populate 'values', consuming options and arguments from 'rargs'. If 'allow_interspersed_args' is false, stop at the first non-option argument. If true, accumulate any interspersed non-option arguments in 'largs'. """ while rargs: arg = rargs[0] # We handle bare "--" explicitly, and bare "-" is handled by the # standard arg handler since the short arg case ensures that the # len of the opt string is greater than 1. if arg == "--": del rargs[0] return elif arg[0:2] == "--": # process a single long option (possibly with value(s)) self._process_long_opt(rargs, values) elif arg[:1] == "-" and len(arg) > 1: # process a cluster of short options (possibly with # value(s) for the last one only) self._process_short_opts(rargs, values) elif self.allow_interspersed_args: largs.append(arg) del rargs[0] else: return # stop now, leave this arg in rargs # Say this is the original argument list: # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)] # ^ # (we are about to process arg(i)). # # Then rargs is [arg(i), ..., arg(N-1)] and largs is a *subset* of # [arg0, ..., arg(i-1)] (any options and their arguments will have # been removed from largs). # # The while loop will usually consume 1 or more arguments per pass. # If it consumes 1 (eg. arg is an option that takes no arguments), # then after _process_arg() is done the situation is: # # largs = subset of [arg0, ..., arg(i)] # rargs = [arg(i+1), ..., arg(N-1)] # # If allow_interspersed_args is false, largs will always be # *empty* -- still a subset of [arg0, ..., arg(i-1)], but # not a very interesting subset! def _match_long_opt(self, opt): """_match_long_opt(opt : string) -> string Determine which long option string 'opt' matches, ie. which one it is an unambiguous abbreviation for. Raises BadOptionError if 'opt' doesn't unambiguously match any long option string. """ return _match_abbrev(opt, self._long_opt) def _process_long_opt(self, rargs, values): # arg = rargs.pop(0) arg = rargs[0] rargs = rargs[1:] # Value explicitly attached to arg? Pretend it's the next # argument. if "=" in arg: (opt, next_arg) = arg.split("=", 1) rargs.insert(0, next_arg) had_explicit_value = True else: opt = arg had_explicit_value = False opt = self._match_long_opt(opt) option = self._long_opt[opt] if option.takes_value(): nargs = option.nargs if len(rargs) < nargs: if nargs == 1: self.error(_("%s option requires an argument") % opt) else: self.error(_("%s option requires %d arguments") % (opt, nargs)) elif nargs == 1: # value = rargs.pop(0) value = rargs[0] rargs = rargs[1:] else: value = tuple(rargs[0:nargs]) del rargs[0:nargs] elif had_explicit_value: self.error(_("%s option does not take a value") % opt) else: value = None option.process(opt, value, values, self) def _process_short_opts(self, rargs, values): # arg = rargs.pop(0) arg = rargs[-1] rargs = rargs[:-1] stop = False i = 1 for ch in arg[1:]: opt = "-" + ch option = self._short_opt.get(opt) i += 1 # we have consumed a character if not option: raise BadOptionError(opt) if option.takes_value(): # Any characters left in arg? Pretend they're the # next arg, and stop consuming characters of arg. if i < len(arg): rargs.insert(0, arg[i:]) stop = True nargs = option.nargs if len(rargs) < nargs: if nargs == 1: self.error(_("%s option requires an argument") % opt) else: self.error(_("%s option requires %d arguments") % (opt, nargs)) elif nargs == 1: # value = rargs.pop(0) value = rargs[0] rargs = rargs[1:] else: value = tuple(rargs[0:nargs]) del rargs[0:nargs] else: # option doesn't take a value value = None option.process(opt, value, values, self) if stop: break # -- Feedback methods ---------------------------------------------- def get_prog_name(self): if self.prog is None: # return os.path.basename(sys.argv[0]) paths = sys.argv[0].split('/') return paths[-1] if paths else '' else: return self.prog def expand_prog_name(self, s): # return s.replace("%prog", self.get_prog_name()) return self.get_prog_name().join(s.split("%prog")) def get_description(self): return self.expand_prog_name(self.description) def exit(self, status=0, msg=None): if msg: sys.stderr.write(msg) sys.exit(status) def error(self, msg): """error(msg : string) Print a usage message incorporating 'msg' to stderr and exit. If you override this in a subclass, it should not return -- it should either exit or raise an exception. """ self.print_usage(sys.stderr) self.exit(2, "%s: error: %s\n" % (self.get_prog_name(), msg)) def get_usage(self): if self.usage: return self.formatter.format_usage( self.expand_prog_name(self.usage)) else: return "" def print_usage(self, file=None): """print_usage(file : file = stdout) Print the usage message for the current program (self.usage) to 'file' (default stdout). Any occurrence of the string "%prog" in self.usage is replaced with the name of the current program (basename of sys.argv[0]). Does nothing if self.usage is empty or not defined. """ if self.usage: print >>file, self.get_usage() def get_version(self): if self.version: return self.expand_prog_name(self.version) else: return "" def print_version(self, file=None): """print_version(file : file = stdout) Print the version message for this program (self.version) to 'file' (default stdout). As with print_usage(), any occurrence of "%prog" in self.version is replaced by the current program's name. Does nothing if self.version is empty or undefined. """ if self.version: print >>file, self.get_version() def format_option_help(self, formatter=None): if formatter is None: formatter = self.formatter formatter.store_option_strings(self) result = [] result.append(formatter.format_heading(_("Options"))) formatter.indent() if self.option_list: result.append(OptionContainer.format_option_help(self, formatter)) result.append("\n") for group in self.option_groups: result.append(group.format_help(formatter)) result.append("\n") formatter.dedent() # Drop the last "\n", or the header if no options or option groups: return "".join(result[:-1]) def format_epilog(self, formatter): return formatter.format_epilog(self.epilog) def format_help(self, formatter=None): if formatter is None: formatter = self.formatter result = [] if self.usage: result.append(self.get_usage() + "\n") if self.description: result.append(self.format_description(formatter) + "\n") result.append(self.format_option_help(formatter)) result.append(self.format_epilog(formatter)) return "".join(result) # used by test suite def _get_encoding(self, file): return "utf-8" # encoding = getattr(file, "encoding", None) # if not encoding: # encoding = sys.getdefaultencoding() # return encoding def print_help(self, file=None): """print_help(file : file = stdout) Print an extended help message, listing all options and any help text provided with them, to 'file' (default stdout). """ if file is None: file = sys.stdout encoding = self._get_encoding(file) # file.write(self.format_help().encode(encoding, "replace")) file.write(self.format_help()) # class OptionParser def _match_abbrev(s, wordmap): """_match_abbrev(s : string, wordmap : {string : Option}) -> string Return the string key in 'wordmap' for which 's' is an unambiguous abbreviation. If 's' is found to be ambiguous or doesn't match any of 'words', raise BadOptionError. """ # Is there an exact match? if s in wordmap: return s else: # Isolate all words with s as a prefix. possibilities = [word for word in wordmap.keys() if word.startswith(s)] # No exact match, so there had better be just one possibility. if len(possibilities) == 1: return possibilities[0] elif not possibilities: raise BadOptionError(s) else: # More than one possible completion: ambiguous prefix. possibilities.sort() raise AmbiguousOptionError(s, possibilities) # Some day, there might be many Option classes. As of Optik 1.3, the # preferred way to instantiate Options is indirectly, via make_option(), # which will become a factory function when there are many Option # classes. make_option = Option ================================================ FILE: third_party/stdlib/pprint.py ================================================ # Author: Fred L. Drake, Jr. # fdrake@acm.org # # This is a simple little module I wrote to make life easier. I didn't # see anything quite like it in the library, though I may have overlooked # something. I wrote this when I was trying to read some heavily nested # tuples with fairly non-descriptive content. This is modeled very much # after Lisp/Scheme - style pretty-printing of lists. If you find it # useful, thank small children who sleep at night. """Support to pretty-print lists, tuples, & dictionaries recursively. Very simple, but useful, especially in debugging data structures. Classes ------- PrettyPrinter() Handle pretty-printing operations onto a stream using a configured set of formatting parameters. Functions --------- pformat() Format a Python object into a pretty-printed representation. pprint() Pretty-print a Python object to a stream [default is sys.stdout]. saferepr() Generate a 'standard' repr()-like value, but protect against recursive data structures. """ import sys as _sys import warnings import StringIO _StringIO = StringIO.StringIO # try: # from cStringIO import StringIO as _StringIO # except ImportError: # from StringIO import StringIO as _StringIO __all__ = ["pprint","pformat","isreadable","isrecursive","saferepr", "PrettyPrinter"] # cache these for faster access: _commajoin = ", ".join _id = id _len = len _type = type def pprint(o, stream=None, indent=1, width=80, depth=None): """Pretty-print a Python o to a stream [default is sys.stdout].""" printer = PrettyPrinter( stream=stream, indent=indent, width=width, depth=depth) printer.pprint(o) def pformat(o, indent=1, width=80, depth=None): """Format a Python o into a pretty-printed representation.""" return PrettyPrinter(indent=indent, width=width, depth=depth).pformat(o) def saferepr(o): """Version of repr() which can handle recursive data structures.""" return _safe_repr(o, {}, None, 0)[0] def isreadable(o): """Determine if saferepr(o) is readable by eval().""" return _safe_repr(o, {}, None, 0)[1] def isrecursive(o): """Determine if o requires a recursive representation.""" return _safe_repr(o, {}, None, 0)[2] def _sorted(iterable): with warnings.catch_warnings(): if _sys.py3kwarning: warnings.filterwarnings("ignore", "comparing unequal types " "not supported", DeprecationWarning) return sorted(iterable) class PrettyPrinter(object): def __init__(self, indent=1, width=80, depth=None, stream=None): """Handle pretty printing operations onto a stream using a set of configured parameters. indent Number of spaces to indent for each level of nesting. width Attempted maximum number of columns in the output. depth The maximum depth to print out nested structures. stream The desired output stream. If omitted (or false), the standard output stream available at construction will be used. """ indent = int(indent) width = int(width) assert indent >= 0, "indent must be >= 0" assert depth is None or depth > 0, "depth must be > 0" assert width, "width must be != 0" self._depth = depth self._indent_per_level = indent self._width = width if stream is not None: self._stream = stream else: self._stream = _sys.stdout def pprint(self, o): self._format(o, self._stream, 0, 0, {}, 0) self._stream.write("\n") def pformat(self, o): sio = _StringIO() self._format(o, sio, 0, 0, {}, 0) return sio.getvalue() def isrecursive(self, o): return self.format(o, {}, 0, 0)[2] def isreadable(self, o): s, readable, recursive = self.format(o, {}, 0, 0) return readable and not recursive def _format(self, o, stream, indent, allowance, context, level): level = level + 1 objid = _id(o) if objid in context: stream.write(_recursion(o)) self._recursive = True self._readable = False return rep = self._repr(o, context, level - 1) typ = _type(o) sepLines = _len(rep) > (self._width - 1 - indent - allowance) write = stream.write if self._depth and level > self._depth: write(rep) return r = getattr(typ, "__repr__", None) if issubclass(typ, dict):# and r == dict.__repr__: write('{') if self._indent_per_level > 1: write((self._indent_per_level - 1) * ' ') length = _len(o) if length: context[objid] = 1 indent = indent + self._indent_per_level items = _sorted(o.items()) key, ent = items[0] rep = self._repr(key, context, level) write(rep) write(': ') self._format(ent, stream, indent + _len(rep) + 2, allowance + 1, context, level) if length > 1: for key, ent in items[1:]: rep = self._repr(key, context, level) if sepLines: write(',\n%s%s: ' % (' '*indent, rep)) else: write(', %s: ' % rep) self._format(ent, stream, indent + _len(rep) + 2, allowance + 1, context, level) indent = indent - self._indent_per_level del context[objid] write('}') return # if ((issubclass(typ, list) and r == list.__repr__) or # (issubclass(typ, tuple) and r == tuple.__repr__) or # (issubclass(typ, set) and r == set.__repr__) or # (issubclass(typ, frozenset) and r == frozenset.__repr__) if ((issubclass(typ, list)) or (issubclass(typ, tuple)) or (issubclass(typ, set)) or (issubclass(typ, frozenset)) ): length = _len(o) if issubclass(typ, list): write('[') endchar = ']' elif issubclass(typ, tuple): write('(') endchar = ')' else: if not length: write(rep) return write(typ.__name__) write('([') endchar = '])' indent += len(typ.__name__) + 1 o = _sorted(o) if self._indent_per_level > 1 and sepLines: write((self._indent_per_level - 1) * ' ') if length: context[objid] = 1 indent = indent + self._indent_per_level self._format(o[0], stream, indent, allowance + 1, context, level) if length > 1: for ent in o[1:]: if sepLines: write(',\n' + ' '*indent) else: write(', ') self._format(ent, stream, indent, allowance + 1, context, level) indent = indent - self._indent_per_level del context[objid] if issubclass(typ, tuple) and length == 1: write(',') write(endchar) return write(rep) def _repr(self, o, context, level): repr1, readable, recursive = self.format(o, dict(context), self._depth, level) if not readable: self._readable = False if recursive: self._recursive = True return repr1 def format(self, o, context, maxlevels, level): """Format o for a specific context, returning a string and flags indicating whether the representation is 'readable' and whether the o represents a recursive construct. """ return _safe_repr(o, context, maxlevels, level) # Return triple (repr_string, isreadable, isrecursive). def _safe_repr(o, context, maxlevels, level): typ = _type(o) if typ is str: # if 'locale' not in _sys.modules: # return repr(o), True, False if "'" in o and '"' not in o: closure = '"' quotes = {'"': '\\"'} else: closure = "'" quotes = {"'": "\\'"} qget = quotes.get sio = _StringIO() write = sio.write for char in o: # if char.isalpha(): if 'a' <= char <= 'z' or 'A' <= char <= 'Z': write(char) else: write(qget(char, repr(char)[1:-1])) return ("%s%s%s" % (closure, sio.getvalue(), closure)), True, False r = getattr(typ, "__repr__", None) if issubclass(typ, dict):# and r == dict.__repr__: if not o: return "{}", True, False objid = _id(o) if maxlevels and level >= maxlevels: return "{...}", False, objid in context if objid in context: return _recursion(o), False, True context[objid] = 1 readable = True recursive = False components = [] append = components.append level += 1 saferepr = _safe_repr for k, v in _sorted(o.items()): krepr, kreadable, krecur = saferepr(k, context, maxlevels, level) vrepr, vreadable, vrecur = saferepr(v, context, maxlevels, level) append("%s: %s" % (krepr, vrepr)) readable = readable and kreadable and vreadable if krecur or vrecur: recursive = True del context[objid] return "{%s}" % _commajoin(components), readable, recursive # if (issubclass(typ, list) and r == list.__repr__) or \ # (issubclass(typ, tuple) and r == tuple.__repr__): if (issubclass(typ, list)) or \ (issubclass(typ, tuple)): if issubclass(typ, list): if not o: return "[]", True, False format = "[%s]" elif _len(o) == 1: format = "(%s,)" else: if not o: return "()", True, False format = "(%s)" objid = _id(o) if maxlevels and level >= maxlevels: return format % "...", False, objid in context if objid in context: return _recursion(o), False, True context[objid] = 1 readable = True recursive = False components = [] append = components.append level += 1 for x in o: orepr, oreadable, orecur = _safe_repr(x, context, maxlevels, level) append(orepr) if not oreadable: readable = False if orecur: recursive = True del context[objid] return format % _commajoin(components), readable, recursive rep = repr(o) return rep, (rep and not rep.startswith('<')), False def _recursion(o): return ("" % (_type(o).__name__, _id(o))) # def _perfcheck(o=None): # import time # if o is None: # o = [("string", (1, 2), [3, 4], {5: 6, 7: 8})] * 100000 # p = PrettyPrinter() # t1 = time.time() # _safe_repr(o, {}, None, 0) # t2 = time.time() # p.pformat(o) # t3 = time.time() # print "_safe_repr:", t2 - t1 # print "pformat:", t3 - t2 # if __name__ == "__main__": # _perfcheck() ================================================ FILE: third_party/stdlib/quopri.py ================================================ #! /usr/bin/env python """Conversions to/from quoted-printable transport encoding as per RFC 1521.""" # (Dec 1991 version). __all__ = ["encode", "decode", "encodestring", "decodestring"] ESCAPE = '=' MAXLINESIZE = 76 HEX = '0123456789ABCDEF' EMPTYSTRING = '' try: from binascii import a2b_qp, b2a_qp except ImportError: a2b_qp = None b2a_qp = None def needsquoting(c, quotetabs, header): """Decide whether a particular character needs to be quoted. The 'quotetabs' flag indicates whether embedded tabs and spaces should be quoted. Note that line-ending tabs and spaces are always encoded, as per RFC 1521. """ if c in ' \t': return quotetabs # if header, we have to escape _ because _ is used to escape space if c == '_': return header return c == ESCAPE or not (' ' <= c <= '~') def quote(c): """Quote a single character.""" i = ord(c) return ESCAPE + HEX[i//16] + HEX[i%16] def encode(input, output, quotetabs, header = 0): """Read 'input', apply quoted-printable encoding, and write to 'output'. 'input' and 'output' are files with readline() and write() methods. The 'quotetabs' flag indicates whether embedded tabs and spaces should be quoted. Note that line-ending tabs and spaces are always encoded, as per RFC 1521. The 'header' flag indicates whether we are encoding spaces as _ as per RFC 1522. """ if b2a_qp is not None: data = input.read() odata = b2a_qp(data, quotetabs = quotetabs, header = header) output.write(odata) return def write(s, output=output, lineEnd='\n'): # RFC 1521 requires that the line ending in a space or tab must have # that trailing character encoded. if s and s[-1:] in ' \t': output.write(s[:-1] + quote(s[-1]) + lineEnd) elif s == '.': output.write(quote(s) + lineEnd) else: output.write(s + lineEnd) prevline = None while 1: line = input.readline() if not line: break outline = [] # Strip off any readline induced trailing newline stripped = '' if line[-1:] == '\n': line = line[:-1] stripped = '\n' # Calculate the un-length-limited encoded line for c in line: if needsquoting(c, quotetabs, header): c = quote(c) if header and c == ' ': outline.append('_') else: outline.append(c) # First, write out the previous line if prevline is not None: write(prevline) # Now see if we need any soft line breaks because of RFC-imposed # length limitations. Then do the thisline->prevline dance. thisline = EMPTYSTRING.join(outline) while len(thisline) > MAXLINESIZE: # Don't forget to include the soft line break `=' sign in the # length calculation! write(thisline[:MAXLINESIZE-1], lineEnd='=\n') thisline = thisline[MAXLINESIZE-1:] # Write out the current line prevline = thisline # Write out the last line, without a trailing newline if prevline is not None: write(prevline, lineEnd=stripped) def encodestring(s, quotetabs = 0, header = 0): if b2a_qp is not None: return b2a_qp(s, quotetabs = quotetabs, header = header) from cStringIO import StringIO infp = StringIO(s) outfp = StringIO() encode(infp, outfp, quotetabs, header) return outfp.getvalue() def decode(input, output, header = 0): """Read 'input', apply quoted-printable decoding, and write to 'output'. 'input' and 'output' are files with readline() and write() methods. If 'header' is true, decode underscore as space (per RFC 1522).""" if a2b_qp is not None: data = input.read() odata = a2b_qp(data, header = header) output.write(odata) return new = '' while 1: line = input.readline() if not line: break i, n = 0, len(line) if n > 0 and line[n-1] == '\n': partial = 0; n = n-1 # Strip trailing whitespace while n > 0 and line[n-1] in " \t\r": n = n-1 else: partial = 1 while i < n: c = line[i] if c == '_' and header: new = new + ' '; i = i+1 elif c != ESCAPE: new = new + c; i = i+1 elif i+1 == n and not partial: partial = 1; break elif i+1 < n and line[i+1] == ESCAPE: new = new + ESCAPE; i = i+2 elif i+2 < n and ishex(line[i+1]) and ishex(line[i+2]): new = new + chr(unhex(line[i+1:i+3])); i = i+3 else: # Bad escape sequence -- leave it in new = new + c; i = i+1 if not partial: output.write(new + '\n') new = '' if new: output.write(new) def decodestring(s, header = 0): if a2b_qp is not None: return a2b_qp(s, header = header) from cStringIO import StringIO infp = StringIO(s) outfp = StringIO() decode(infp, outfp, header = header) return outfp.getvalue() # Other helper functions def ishex(c): """Return true if the character 'c' is a hexadecimal digit.""" return '0' <= c <= '9' or 'a' <= c <= 'f' or 'A' <= c <= 'F' def unhex(s): """Get the integer value of a hexadecimal number.""" bits = 0 for c in s: if '0' <= c <= '9': i = ord('0') elif 'a' <= c <= 'f': i = ord('a')-10 elif 'A' <= c <= 'F': i = ord('A')-10 else: break bits = bits*16 + (ord(c) - i) return bits def main(): import sys import getopt try: opts, args = getopt.getopt(sys.argv[1:], 'td') except getopt.error, msg: sys.stdout = sys.stderr print msg print "usage: quopri [-t | -d] [file] ..." print "-t: quote tabs" print "-d: decode; default encode" sys.exit(2) deco = 0 tabs = 0 for o, a in opts: if o == '-t': tabs = 1 if o == '-d': deco = 1 if tabs and deco: sys.stdout = sys.stderr print "-t and -d are mutually exclusive" sys.exit(2) if not args: args = ['-'] sts = 0 for file in args: if file == '-': fp = sys.stdin else: try: fp = open(file) except IOError, msg: sys.stderr.write("%s: can't open (%s)\n" % (file, msg)) sts = 1 continue if deco: decode(fp, sys.stdout) else: encode(fp, sys.stdout, tabs) if fp is not sys.stdin: fp.close() if sts: sys.exit(sts) if __name__ == '__main__': main() ================================================ FILE: third_party/stdlib/random.py ================================================ """Random variable generators. integers -------- uniform within range sequences --------- pick random element pick random sample generate random permutation distributions on the real line: ------------------------------ uniform triangular normal (Gaussian) lognormal negative exponential gamma beta pareto Weibull distributions on the circle (angles 0 to 2pi) --------------------------------------------- circular uniform von Mises General notes on the underlying Mersenne Twister core generator: * The period is 2**19937-1. * It is one of the most extensively tested generators in existence. * Without a direct way to compute N steps forward, the semantics of jumpahead(n) are weakened to simply jump to another distant state and rely on the large period to avoid overlapping sequences. * The random() method is implemented in C, executes in a single Python step, and is, therefore, threadsafe. """ #from warnings import warn as _warn #from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethodType #from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil #from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin #from os import urandom as _urandom #from binascii import hexlify as _hexlify #import hashlib as _hashlib __all__ = ["Random","seed","random","uniform","randint","choice","sample", "randrange","shuffle","normalvariate","lognormvariate", "expovariate","vonmisesvariate","gammavariate","triangular", "gauss","betavariate","paretovariate","weibullvariate", "getstate","setstate","jumpahead", "WichmannHill", "getrandbits", "SystemRandom"] # Import _random wrapper from grumpy std lib. # It is used as a replacement for the CPython random generator. import _random # NV_MAGICCONST = 4 * _exp(-0.5)/_sqrt(2.0) # TWOPI = 2.0*_pi # LOG4 = _log(4.0) # SG_MAGICCONST = 1.0 + _log(4.5) BPF = _random.BPF RECIP_BPF = _random.RECIP_BPF class Random(_random.GrumpyRandom): """Random number generator base class used by bound module functions. Used to instantiate instances of Random to get generators that don't share state. Especially useful for multi-threaded programs, creating a different instance of Random for each thread, and using the jumpahead() method to ensure that the generated sequences seen by each thread don't overlap. Class Random can also be subclassed if you want to use a different basic generator of your own devising: in that case, override the following methods: random(), seed(), getstate(), setstate() and jumpahead(). Optionally, implement a getrandbits() method so that randrange() can cover arbitrarily large ranges. """ VERSION = 3 # used by getstate/setstate def __init__(self, x=None): """Initialize an instance. Optional argument x controls seeding, as for Random.seed(). """ self.seed(x) self.gauss_next = None def seed(self, a=None): """Initialize internal state of the random number generator. None or no argument seeds from current time or from an operating system specific randomness source if available. If a is not None or is an int or long, hash(a) is used instead. Hash values for some types are nondeterministic when the PYTHONHASHSEED environment variable is enabled. """ super(Random, self).seed(a) self.gauss_next = None ## ---- Methods below this point do not need to be overridden when ## ---- subclassing for the purpose of using a different core generator. ## -------------------- integer methods ------------------- def randrange(self, start, stop=None, step=1, _int=int, _maxwidth=1L< 0: if istart >= _maxwidth: return self._randbelow(istart) return _int(self.random() * istart) raise ValueError, "empty range for randrange()" # stop argument supplied. istop = _int(stop) if istop != stop: raise ValueError, "non-integer stop for randrange()" width = istop - istart if step == 1 and width > 0: # Note that # int(istart + self.random()*width) # instead would be incorrect. For example, consider istart # = -2 and istop = 0. Then the guts would be in # -2.0 to 0.0 exclusive on both ends (ignoring that random() # might return 0.0), and because int() truncates toward 0, the # final result would be -1 or 0 (instead of -2 or -1). # istart + int(self.random()*width) # would also be incorrect, for a subtler reason: the RHS # can return a long, and then randrange() would also return # a long, but we're supposed to return an int (for backward # compatibility). if width >= _maxwidth: return _int(istart + self._randbelow(width)) return _int(istart + _int(self.random()*width)) if step == 1: raise ValueError, "empty range for randrange() (%d,%d, %d)" % (istart, istop, width) # Non-unit step argument supplied. istep = _int(step) if istep != step: raise ValueError, "non-integer step for randrange()" if istep > 0: n = (width + istep - 1) // istep elif istep < 0: n = (width + istep + 1) // istep else: raise ValueError, "zero step for randrange()" if n <= 0: raise ValueError, "empty range for randrange()" if n >= _maxwidth: return istart + istep*self._randbelow(n) return istart + istep*_int(self.random() * n) def randint(self, a, b): """Return random integer in range [a, b], including both end points. """ return self.randrange(a, b+1) ## -------------------- sequence methods ------------------- def choice(self, seq): """Choose a random element from a non-empty sequence.""" return seq[int(self.random() * len(seq))] # raises IndexError if seq is empty def shuffle(self, x, random=None): """x, random=random.random -> shuffle list x in place; return None. Optional arg random is a 0-argument function returning a random float in [0.0, 1.0); by default, the standard random.random. """ if random is None: random = self.random _int = int for i in reversed(xrange(1, len(x))): # pick an element in x[:i+1] with which to exchange x[i] j = _int(random() * (i+1)) x[i], x[j] = x[j], x[i] # def sample(self, population, k): # """Chooses k unique random elements from a population sequence. # Returns a new list containing elements from the population while # leaving the original population unchanged. The resulting list is # in selection order so that all sub-slices will also be valid random # samples. This allows raffle winners (the sample) to be partitioned # into grand prize and second place winners (the subslices). # Members of the population need not be hashable or unique. If the # population contains repeats, then each occurrence is a possible # selection in the sample. # To choose a sample in a range of integers, use xrange as an argument. # This is especially fast and space efficient for sampling from a # large population: sample(xrange(10000000), 60) # """ # # Sampling without replacement entails tracking either potential # # selections (the pool) in a list or previous selections in a set. # # When the number of selections is small compared to the # # population, then tracking selections is efficient, requiring # # only a small set and an occasional reselection. For # # a larger number of selections, the pool tracking method is # # preferred since the list takes less space than the # # set and it doesn't suffer from frequent reselections. # n = len(population) # if not 0 <= k <= n: # raise ValueError("sample larger than population") # random = self.random # _int = int # result = [None] * k # setsize = 21 # size of a small set minus size of an empty list # if k > 5: # setsize += 4 ** _ceil(_log(k * 3, 4)) # table size for big sets # if n <= setsize or hasattr(population, "keys"): # # An n-length list is smaller than a k-length set, or this is a # # mapping type so the other algorithm wouldn't work. # pool = list(population) # for i in xrange(k): # invariant: non-selected at [0,n-i) # j = _int(random() * (n-i)) # result[i] = pool[j] # pool[j] = pool[n-i-1] # move non-selected item into vacancy # else: # try: # selected = set() # selected_add = selected.add # for i in xrange(k): # j = _int(random() * n) # while j in selected: # j = _int(random() * n) # selected_add(j) # result[i] = population[j] # except (TypeError, KeyError): # handle (at least) sets # if isinstance(population, list): # raise # return self.sample(tuple(population), k) # return result ## -------------------- real-valued distributions ------------------- ## -------------------- uniform distribution ------------------- def uniform(self, a, b): "Get a random number in the range [a, b) or [a, b] depending on rounding." return a + (b-a) * self.random() ## -------------------- triangular -------------------- # def triangular(self, low=0.0, high=1.0, mode=None): # """Triangular distribution. # Continuous distribution bounded by given lower and upper limits, # and having a given mode value in-between. # http://en.wikipedia.org/wiki/Triangular_distribution # """ # u = self.random() # try: # c = 0.5 if mode is None else (mode - low) / (high - low) # except ZeroDivisionError: # return low # if u > c: # u = 1.0 - u # c = 1.0 - c # low, high = high, low # return low + (high - low) * (u * c) ** 0.5 ## -------------------- normal distribution -------------------- # def normalvariate(self, mu, sigma): # """Normal distribution. # mu is the mean, and sigma is the standard deviation. # """ # # mu = mean, sigma = standard deviation # # Uses Kinderman and Monahan method. Reference: Kinderman, # # A.J. and Monahan, J.F., "Computer generation of random # # variables using the ratio of uniform deviates", ACM Trans # # Math Software, 3, (1977), pp257-260. # random = self.random # while 1: # u1 = random() # u2 = 1.0 - random() # z = NV_MAGICCONST*(u1-0.5)/u2 # zz = z*z/4.0 # if zz <= -_log(u2): # break # return mu + z*sigma ## -------------------- lognormal distribution -------------------- # def lognormvariate(self, mu, sigma): # """Log normal distribution. # If you take the natural logarithm of this distribution, you'll get a # normal distribution with mean mu and standard deviation sigma. # mu can have any value, and sigma must be greater than zero. # """ # return _exp(self.normalvariate(mu, sigma)) ## -------------------- exponential distribution -------------------- # def expovariate(self, lambd): # """Exponential distribution. # lambd is 1.0 divided by the desired mean. It should be # nonzero. (The parameter would be called "lambda", but that is # a reserved word in Python.) Returned values range from 0 to # positive infinity if lambd is positive, and from negative # infinity to 0 if lambd is negative. # """ # # lambd: rate lambd = 1/mean # # ('lambda' is a Python reserved word) # # we use 1-random() instead of random() to preclude the # # possibility of taking the log of zero. # return -_log(1.0 - self.random())/lambd ## -------------------- von Mises distribution -------------------- # def vonmisesvariate(self, mu, kappa): # """Circular data distribution. # mu is the mean angle, expressed in radians between 0 and 2*pi, and # kappa is the concentration parameter, which must be greater than or # equal to zero. If kappa is equal to zero, this distribution reduces # to a uniform random angle over the range 0 to 2*pi. # """ # # mu: mean angle (in radians between 0 and 2*pi) # # kappa: concentration parameter kappa (>= 0) # # if kappa = 0 generate uniform random angle # # Based upon an algorithm published in: Fisher, N.I., # # "Statistical Analysis of Circular Data", Cambridge # # University Press, 1993. # # Thanks to Magnus Kessler for a correction to the # # implementation of step 4. # random = self.random # if kappa <= 1e-6: # return TWOPI * random() # s = 0.5 / kappa # r = s + _sqrt(1.0 + s * s) # while 1: # u1 = random() # z = _cos(_pi * u1) # d = z / (r + z) # u2 = random() # if u2 < 1.0 - d * d or u2 <= (1.0 - d) * _exp(d): # break # q = 1.0 / r # f = (q + z) / (1.0 + q * z) # u3 = random() # if u3 > 0.5: # theta = (mu + _acos(f)) % TWOPI # else: # theta = (mu - _acos(f)) % TWOPI # return theta ## -------------------- gamma distribution -------------------- # def gammavariate(self, alpha, beta): # """Gamma distribution. Not the gamma function! # Conditions on the parameters are alpha > 0 and beta > 0. # The probability distribution function is: # x ** (alpha - 1) * math.exp(-x / beta) # pdf(x) = -------------------------------------- # math.gamma(alpha) * beta ** alpha # """ # # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 # # Warning: a few older sources define the gamma distribution in terms # # of alpha > -1.0 # if alpha <= 0.0 or beta <= 0.0: # raise ValueError, 'gammavariate: alpha and beta must be > 0.0' # random = self.random # if alpha > 1.0: # # Uses R.C.H. Cheng, "The generation of Gamma # # variables with non-integral shape parameters", # # Applied Statistics, (1977), 26, No. 1, p71-74 # ainv = _sqrt(2.0 * alpha - 1.0) # bbb = alpha - LOG4 # ccc = alpha + ainv # while 1: # u1 = random() # if not 1e-7 < u1 < .9999999: # continue # u2 = 1.0 - random() # v = _log(u1/(1.0-u1))/ainv # x = alpha*_exp(v) # z = u1*u1*u2 # r = bbb+ccc*v-x # if r + SG_MAGICCONST - 4.5*z >= 0.0 or r >= _log(z): # return x * beta # elif alpha == 1.0: # # expovariate(1) # u = random() # while u <= 1e-7: # u = random() # return -_log(u) * beta # else: # alpha is between 0 and 1 (exclusive) # # Uses ALGORITHM GS of Statistical Computing - Kennedy & Gentle # while 1: # u = random() # b = (_e + alpha)/_e # p = b*u # if p <= 1.0: # x = p ** (1.0/alpha) # else: # x = -_log((b-p)/alpha) # u1 = random() # if p > 1.0: # if u1 <= x ** (alpha - 1.0): # break # elif u1 <= _exp(-x): # break # return x * beta ## -------------------- Gauss (faster alternative) -------------------- # def gauss(self, mu, sigma): # """Gaussian distribution. # mu is the mean, and sigma is the standard deviation. This is # slightly faster than the normalvariate() function. # Not thread-safe without a lock around calls. # """ # # When x and y are two variables from [0, 1), uniformly # # distributed, then # # # # cos(2*pi*x)*sqrt(-2*log(1-y)) # # sin(2*pi*x)*sqrt(-2*log(1-y)) # # # # are two *independent* variables with normal distribution # # (mu = 0, sigma = 1). # # (Lambert Meertens) # # (corrected version; bug discovered by Mike Miller, fixed by LM) # # Multithreading note: When two threads call this function # # simultaneously, it is possible that they will receive the # # same return value. The window is very small though. To # # avoid this, you have to use a lock around all calls. (I # # didn't want to slow this down in the serial case by using a # # lock here.) # random = self.random # z = self.gauss_next # self.gauss_next = None # if z is None: # x2pi = random() * TWOPI # g2rad = _sqrt(-2.0 * _log(1.0 - random())) # z = _cos(x2pi) * g2rad # self.gauss_next = _sin(x2pi) * g2rad # return mu + z*sigma ## -------------------- beta -------------------- ## See ## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html ## for Ivan Frohne's insightful analysis of why the original implementation: ## ## def betavariate(self, alpha, beta): ## # Discrete Event Simulation in C, pp 87-88. ## ## y = self.expovariate(alpha) ## z = self.expovariate(1.0/beta) ## return z/(y+z) ## ## was dead wrong, and how it probably got that way. # def betavariate(self, alpha, beta): # """Beta distribution. # Conditions on the parameters are alpha > 0 and beta > 0. # Returned values range between 0 and 1. # """ # # This version due to Janne Sinkkonen, and matches all the std # # texts (e.g., Knuth Vol 2 Ed 3 pg 134 "the beta distribution"). # y = self.gammavariate(alpha, 1.) # if y == 0: # return 0.0 # else: # return y / (y + self.gammavariate(beta, 1.)) ## -------------------- Pareto -------------------- # def paretovariate(self, alpha): # """Pareto distribution. alpha is the shape parameter.""" # # Jain, pg. 495 # u = 1.0 - self.random() # return 1.0 / pow(u, 1.0/alpha) ## -------------------- Weibull -------------------- # def weibullvariate(self, alpha, beta): # """Weibull distribution. # alpha is the scale parameter and beta is the shape parameter. # """ # # Jain, pg. 499; bug fix courtesy Bill Arms # u = 1.0 - self.random() # return alpha * pow(-_log(u), 1.0/beta) ## -------------------- test program -------------------- def _test_generator(n, func, args): import time print n, 'times', func.__name__ total = 0.0 sqsum = 0.0 smallest = 1e10 largest = -1e10 t0 = time.time() for i in range(n): x = func(*args) total += x sqsum = sqsum + x*x smallest = min(x, smallest) largest = max(x, largest) t1 = time.time() print round(t1-t0, 3), 'sec,', avg = total/n stddev = _sqrt(sqsum/n - avg*avg) print 'avg %g, stddev %g, min %g, max %g' % \ (avg, stddev, smallest, largest) def _test(N=2000): _test_generator(N, random, ()) _test_generator(N, normalvariate, (0.0, 1.0)) _test_generator(N, lognormvariate, (0.0, 1.0)) _test_generator(N, vonmisesvariate, (0.0, 1.0)) _test_generator(N, gammavariate, (0.01, 1.0)) _test_generator(N, gammavariate, (0.1, 1.0)) _test_generator(N, gammavariate, (0.1, 2.0)) _test_generator(N, gammavariate, (0.5, 1.0)) _test_generator(N, gammavariate, (0.9, 1.0)) _test_generator(N, gammavariate, (1.0, 1.0)) _test_generator(N, gammavariate, (2.0, 1.0)) _test_generator(N, gammavariate, (20.0, 1.0)) _test_generator(N, gammavariate, (200.0, 1.0)) _test_generator(N, gauss, (0.0, 1.0)) _test_generator(N, betavariate, (3.0, 3.0)) _test_generator(N, triangular, (0.0, 1.0, 1.0/3.0)) # Create one instance, seeded from current time, and export its methods # as module-level functions. The functions share state across all uses #(both in the user's code and in the Python libraries), but that's fine # for most programs and is easier for the casual user than making them # instantiate their own Random() instance. _inst = Random() seed = _inst.seed random = _inst.random randint = _inst.randint choice = _inst.choice randrange = _inst.randrange getrandbits = _inst.getrandbits getstate = _inst.getstate setstate = _inst.setstate uniform = _inst.uniform def _notimplemented(*args, **kwargs): raise NotImplementedError shuffle = _notimplemented choices = _notimplemented sample = _notimplemented triangular = _notimplemented normalvariate = _notimplemented lognormvariate = _notimplemented expovariate = _notimplemented vonmisesvariate = _notimplemented gammavariate = _notimplemented gauss = _notimplemented betavariate = _notimplemented paretovariate = _notimplemented weibullvariate = _notimplemented if __name__ == '__main__': pass #_test() ================================================ FILE: third_party/stdlib/re.py ================================================ # # Secret Labs' Regular Expression Engine # # re-compatible interface for the sre matching engine # # Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. # # This version of the SRE library can be redistributed under CNRI's # Python 1.6 license. For any other use, please contact Secret Labs # AB (info@pythonware.com). # # Portions of this engine have been developed in cooperation with # CNRI. Hewlett-Packard provided funding for 1.6 integration and # other compatibility work. # r"""Support for regular expressions (RE). This module provides regular expression matching operations similar to those found in Perl. It supports both 8-bit and Unicode strings; both the pattern and the strings being processed can contain null bytes and characters outside the US ASCII range. Regular expressions can contain both special and ordinary characters. Most ordinary characters, like "A", "a", or "0", are the simplest regular expressions; they simply match themselves. You can concatenate ordinary characters, so last matches the string 'last'. The special characters are: "." Matches any character except a newline. "^" Matches the start of the string. "$" Matches the end of the string or just before the newline at the end of the string. "*" Matches 0 or more (greedy) repetitions of the preceding RE. Greedy means that it will match as many repetitions as possible. "+" Matches 1 or more (greedy) repetitions of the preceding RE. "?" Matches 0 or 1 (greedy) of the preceding RE. *?,+?,?? Non-greedy versions of the previous three special characters. {m,n} Matches from m to n repetitions of the preceding RE. {m,n}? Non-greedy version of the above. "\\" Either escapes special characters or signals a special sequence. [] Indicates a set of characters. A "^" as the first character indicates a complementing set. "|" A|B, creates an RE that will match either A or B. (...) Matches the RE inside the parentheses. The contents can be retrieved or matched later in the string. (?iLmsux) Set the I, L, M, S, U, or X flag for the RE (see below). (?:...) Non-grouping version of regular parentheses. (?P...) The substring matched by the group is accessible by name. (?P=name) Matches the text matched earlier by the group named name. (?#...) A comment; ignored. (?=...) Matches if ... matches next, but doesn't consume the string. (?!...) Matches if ... doesn't match next. (?<=...) Matches if preceded by ... (must be fixed length). (?= 0x02020000: # __all__.append("finditer") def finditer(pattern, string, flags=0): """Return an iterator over all non-overlapping matches in the string. For each match, the iterator returns a match object. Empty matches are included in the result.""" return _compile(pattern, flags).finditer(string) def compile(pattern, flags=0): "Compile a regular expression pattern, returning a pattern object." return _compile(pattern, flags) def purge(): "Clear the regular expression cache" _cache.clear() _cache_repl.clear() def template(pattern, flags=0): "Compile a template pattern, returning a pattern object" return _compile(pattern, flags|T) _alphanum = frozenset( "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") def escape(pattern): "Escape all non-alphanumeric characters in pattern." s = list(pattern) alphanum = _alphanum for i, c in enumerate(pattern): if c not in alphanum: if c == "\000": s[i] = "\\000" else: s[i] = "\\" + c return pattern[:0].join(s) # -------------------------------------------------------------------- # internals _cache = {} _cache_repl = {} _pattern_type = type(sre_compile.compile("", 0)) _MAXCACHE = 100 def _compile(*key): # internal: compile pattern pattern, flags = key bypass_cache = flags & DEBUG if not bypass_cache: cachekey = (type(key[0]),) + key # try: # p, loc = _cache[cachekey] # if loc is None or loc == _locale.setlocale(_locale.LC_CTYPE): # return p # except KeyError: # pass if isinstance(pattern, _pattern_type): if flags: raise ValueError('Cannot process flags argument with a compiled pattern') return pattern if not sre_compile.isstring(pattern): raise TypeError, "first argument must be string or compiled pattern" try: p = sre_compile.compile(pattern, flags) except error, v: raise error, v # invalid expression if not bypass_cache: if len(_cache) >= _MAXCACHE: _cache.clear() if p.flags & LOCALE: if not _locale: return p # loc = _locale.setlocale(_locale.LC_CTYPE) else: loc = None _cache[cachekey] = p, loc return p def _compile_repl(*key): # internal: compile replacement pattern p = _cache_repl.get(key) if p is not None: return p repl, pattern = key try: p = sre_parse.parse_template(repl, pattern) except error, v: raise error, v # invalid expression if len(_cache_repl) >= _MAXCACHE: _cache_repl.clear() _cache_repl[key] = p return p def _expand(pattern, match, template): # internal: match.expand implementation hook template = sre_parse.parse_template(template, pattern) return sre_parse.expand_template(template, match) def _subx(pattern, template): # internal: pattern.sub/subn implementation helper template = _compile_repl(template, pattern) if not template[0] and len(template[1]) == 1: # literal replacement return template[1][0] def filter(match, template=template): return sre_parse.expand_template(template, match) return filter # register myself for pickling import copy_reg def _pickle(p): return _compile, (p.pattern, p.flags) copy_reg.pickle(_pattern_type, _pickle, _compile) # -------------------------------------------------------------------- # experimental stuff (see python-dev discussions for details) class Scanner(object): def __init__(self, lexicon, flags=0): self.lexicon = lexicon # combine phrases into a compound pattern p = [] s = sre_parse.Pattern() s.flags = flags for phrase, action in lexicon: p.append(sre_parse.SubPattern(s, [ (SUBPATTERN, (len(p)+1, sre_parse.parse(phrase, flags))), ])) s.groups = len(p)+1 p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) self.scanner = sre_compile.compile(p) def scan(self, string): result = [] append = result.append match = self.scanner.scanner(string).match i = 0 while 1: m = match() if not m: break j = m.end() if i == j: break action = self.lexicon[m.lastindex-1][1] if hasattr(action, '__call__'): self.match = m action = action(self, m.group()) if action is not None: append(action) i = j return result, string[i:] ================================================ FILE: third_party/stdlib/re_tests.py ================================================ #!/usr/bin/env python # -*- mode: python -*- import re # Re test suite and benchmark suite v1.5 # The 3 possible outcomes for each pattern [SUCCEED, FAIL, SYNTAX_ERROR] = range(3) # Benchmark suite (needs expansion) # # The benchmark suite does not test correctness, just speed. The # first element of each tuple is the regex pattern; the second is a # string to match it against. The benchmarking code will embed the # second string inside several sizes of padding, to test how regex # matching performs on large strings. # benchmarks = [ # # test common prefix # ('Python|Perl', 'Perl'), # Alternation # ('(Python|Perl)', 'Perl'), # Grouped alternation # ('Python|Perl|Tcl', 'Perl'), # Alternation # ('(Python|Perl|Tcl)', 'Perl'), # Grouped alternation # ('(Python)\\1', 'PythonPython'), # Backreference # ('([0a-z][a-z0-9]*,)+', 'a5,b7,c9,'), # Disable the fastmap optimization # ('([a-z][a-z0-9]*,)+', 'a5,b7,c9,'), # A few sets # ('Python', 'Python'), # Simple text literal # ('.*Python', 'Python'), # Bad text literal # ('.*Python.*', 'Python'), # Worse text literal # ('.*(Python)', 'Python'), # Bad text literal with grouping # ] # Test suite (for verifying correctness) # # The test suite is a list of 5- or 3-tuples. The 5 parts of a # complete tuple are: # element 0: a string containing the pattern # 1: the string to match against the pattern # 2: the expected result (SUCCEED, FAIL, SYNTAX_ERROR) # 3: a string that will be eval()'ed to produce a test string. # This is an arbitrary Python expression; the available # variables are "found" (the whole match), and "g1", "g2", ... # up to "g99" contain the contents of each group, or the # string 'None' if the group wasn't given a value, or the # string 'Error' if the group index was out of range; # also "groups", the return value of m.group() (a tuple). # 4: The expected result of evaluating the expression. # If the two don't match, an error is reported. # # If the regex isn't expected to work, the latter two elements can be omitted. tests = [ # Test ?P< and ?P= extensions ('(?Pa)', '', SYNTAX_ERROR), # Begins with a digit ('(?Pa)', '', SYNTAX_ERROR), # Begins with an illegal char ('(?Pa)', '', SYNTAX_ERROR), # Begins with an illegal char # Same tests, for the ?P= form ('(?Pa)(?P=foo_123', 'aa', SYNTAX_ERROR), ('(?Pa)(?P=1)', 'aa', SYNTAX_ERROR), ('(?Pa)(?P=!)', 'aa', SYNTAX_ERROR), ('(?Pa)(?P=foo_124', 'aa', SYNTAX_ERROR), # Backref to undefined group ('(?Pa)', 'a', SUCCEED, 'g1', 'a'), ('(?Pa)(?P=foo_123)', 'aa', SUCCEED, 'g1', 'a'), # Test octal escapes ('\\1', 'a', SYNTAX_ERROR), # Backreference ('[\\1]', '\1', SUCCEED, 'found', '\1'), # Character ('\\09', chr(0) + '9', SUCCEED, 'found', chr(0) + '9'), ('\\141', 'a', SUCCEED, 'found', 'a'), ('(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)\\119', 'abcdefghijklk9', SUCCEED, 'found+"-"+g11', 'abcdefghijklk9-k'), # Test \0 is handled everywhere (r'\0', '\0', SUCCEED, 'found', '\0'), (r'[\0a]', '\0', SUCCEED, 'found', '\0'), (r'[a\0]', '\0', SUCCEED, 'found', '\0'), (r'[^a\0]', '\0', FAIL), # Test various letter escapes (r'\a[\b]\f\n\r\t\v', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'), (r'[\a][\b][\f][\n][\r][\t][\v]', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'), # NOTE: not an error under PCRE/PRE: # (r'\u', '', SYNTAX_ERROR), # A Perl escape (r'\c\e\g\h\i\j\k\m\o\p\q\y\z', 'ceghijkmopqyz', SUCCEED, 'found', 'ceghijkmopqyz'), (r'\xff', '\377', SUCCEED, 'found', chr(255)), # new \x semantics (r'\x00ffffffffffffff', '\377', FAIL, 'found', chr(255)), (r'\x00f', '\017', FAIL, 'found', chr(15)), (r'\x00fe', '\376', FAIL, 'found', chr(254)), # (r'\x00ffffffffffffff', '\377', SUCCEED, 'found', chr(255)), # (r'\x00f', '\017', SUCCEED, 'found', chr(15)), # (r'\x00fe', '\376', SUCCEED, 'found', chr(254)), (r"^\w+=(\\[\000-\277]|[^\n\\])*", "SRC=eval.c g.c blah blah blah \\\\\n\tapes.c", SUCCEED, 'found', "SRC=eval.c g.c blah blah blah \\\\"), # Test that . only matches \n in DOTALL mode ('a.b', 'acb', SUCCEED, 'found', 'acb'), ('a.b', 'a\nb', FAIL), ('a.*b', 'acc\nccb', FAIL), ('a.{4,5}b', 'acc\nccb', FAIL), ('a.b', 'a\rb', SUCCEED, 'found', 'a\rb'), ('a.b(?s)', 'a\nb', SUCCEED, 'found', 'a\nb'), ('a.*(?s)b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'), ('(?s)a.{4,5}b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'), ('(?s)a.b', 'a\nb', SUCCEED, 'found', 'a\nb'), (')', '', SYNTAX_ERROR), # Unmatched right bracket ('', '', SUCCEED, 'found', ''), # Empty pattern ('abc', 'abc', SUCCEED, 'found', 'abc'), ('abc', 'xbc', FAIL), ('abc', 'axc', FAIL), ('abc', 'abx', FAIL), ('abc', 'xabcy', SUCCEED, 'found', 'abc'), ('abc', 'ababc', SUCCEED, 'found', 'abc'), ('ab*c', 'abc', SUCCEED, 'found', 'abc'), ('ab*bc', 'abc', SUCCEED, 'found', 'abc'), ('ab*bc', 'abbc', SUCCEED, 'found', 'abbc'), ('ab*bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'), ('ab+bc', 'abbc', SUCCEED, 'found', 'abbc'), ('ab+bc', 'abc', FAIL), ('ab+bc', 'abq', FAIL), ('ab+bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'), ('ab?bc', 'abbc', SUCCEED, 'found', 'abbc'), ('ab?bc', 'abc', SUCCEED, 'found', 'abc'), ('ab?bc', 'abbbbc', FAIL), ('ab?c', 'abc', SUCCEED, 'found', 'abc'), ('^abc$', 'abc', SUCCEED, 'found', 'abc'), ('^abc$', 'abcc', FAIL), ('^abc', 'abcc', SUCCEED, 'found', 'abc'), ('^abc$', 'aabc', FAIL), ('abc$', 'aabc', SUCCEED, 'found', 'abc'), ('^', 'abc', SUCCEED, 'found+"-"', '-'), ('$', 'abc', SUCCEED, 'found+"-"', '-'), ('a.c', 'abc', SUCCEED, 'found', 'abc'), ('a.c', 'axc', SUCCEED, 'found', 'axc'), ('a.*c', 'axyzc', SUCCEED, 'found', 'axyzc'), ('a.*c', 'axyzd', FAIL), ('a[bc]d', 'abc', FAIL), ('a[bc]d', 'abd', SUCCEED, 'found', 'abd'), ('a[b-d]e', 'abd', FAIL), ('a[b-d]e', 'ace', SUCCEED, 'found', 'ace'), ('a[b-d]', 'aac', SUCCEED, 'found', 'ac'), ('a[-b]', 'a-', SUCCEED, 'found', 'a-'), ('a[\\-b]', 'a-', SUCCEED, 'found', 'a-'), # NOTE: not an error under PCRE/PRE: # ('a[b-]', 'a-', SYNTAX_ERROR), ('a[]b', '-', SYNTAX_ERROR), ('a[', '-', SYNTAX_ERROR), ('a\\', '-', SYNTAX_ERROR), ('abc)', '-', SYNTAX_ERROR), ('(abc', '-', SYNTAX_ERROR), ('a]', 'a]', SUCCEED, 'found', 'a]'), ('a[]]b', 'a]b', SUCCEED, 'found', 'a]b'), ('a[\]]b', 'a]b', SUCCEED, 'found', 'a]b'), ('a[^bc]d', 'aed', SUCCEED, 'found', 'aed'), ('a[^bc]d', 'abd', FAIL), ('a[^-b]c', 'adc', SUCCEED, 'found', 'adc'), ('a[^-b]c', 'a-c', FAIL), ('a[^]b]c', 'a]c', FAIL), ('a[^]b]c', 'adc', SUCCEED, 'found', 'adc'), ('\\ba\\b', 'a-', SUCCEED, '"-"', '-'), ('\\ba\\b', '-a', SUCCEED, '"-"', '-'), ('\\ba\\b', '-a-', SUCCEED, '"-"', '-'), ('\\by\\b', 'xy', FAIL), ('\\by\\b', 'yz', FAIL), ('\\by\\b', 'xyz', FAIL), ('x\\b', 'xyz', FAIL), ('x\\B', 'xyz', SUCCEED, '"-"', '-'), ('\\Bz', 'xyz', SUCCEED, '"-"', '-'), ('z\\B', 'xyz', FAIL), ('\\Bx', 'xyz', FAIL), ('\\Ba\\B', 'a-', FAIL, '"-"', '-'), ('\\Ba\\B', '-a', FAIL, '"-"', '-'), ('\\Ba\\B', '-a-', FAIL, '"-"', '-'), ('\\By\\B', 'xy', FAIL), ('\\By\\B', 'yz', FAIL), ('\\By\\b', 'xy', SUCCEED, '"-"', '-'), ('\\by\\B', 'yz', SUCCEED, '"-"', '-'), ('\\By\\B', 'xyz', SUCCEED, '"-"', '-'), ('ab|cd', 'abc', SUCCEED, 'found', 'ab'), ('ab|cd', 'abcd', SUCCEED, 'found', 'ab'), ('()ef', 'def', SUCCEED, 'found+"-"+g1', 'ef-'), ('$b', 'b', FAIL), ('a\\(b', 'a(b', SUCCEED, 'found+"-"+g1', 'a(b-Error'), ('a\\(*b', 'ab', SUCCEED, 'found', 'ab'), ('a\\(*b', 'a((b', SUCCEED, 'found', 'a((b'), ('a\\\\b', 'a\\b', SUCCEED, 'found', 'a\\b'), ('((a))', 'abc', SUCCEED, 'found+"-"+g1+"-"+g2', 'a-a-a'), ('(a)b(c)', 'abc', SUCCEED, 'found+"-"+g1+"-"+g2', 'abc-a-c'), ('a+b+c', 'aabbabc', SUCCEED, 'found', 'abc'), ('(a+|b)*', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'), ('(a+|b)+', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'), ('(a+|b)?', 'ab', SUCCEED, 'found+"-"+g1', 'a-a'), (')(', '-', SYNTAX_ERROR), ('[^ab]*', 'cde', SUCCEED, 'found', 'cde'), ('abc', '', FAIL), ('a*', '', SUCCEED, 'found', ''), ('a|b|c|d|e', 'e', SUCCEED, 'found', 'e'), ('(a|b|c|d|e)f', 'ef', SUCCEED, 'found+"-"+g1', 'ef-e'), ('abcd*efg', 'abcdefg', SUCCEED, 'found', 'abcdefg'), ('ab*', 'xabyabbbz', SUCCEED, 'found', 'ab'), ('ab*', 'xayabbbz', SUCCEED, 'found', 'a'), ('(ab|cd)e', 'abcde', SUCCEED, 'found+"-"+g1', 'cde-cd'), ('[abhgefdc]ij', 'hij', SUCCEED, 'found', 'hij'), ('^(ab|cd)e', 'abcde', FAIL, 'xg1y', 'xy'), ('(abc|)ef', 'abcdef', SUCCEED, 'found+"-"+g1', 'ef-'), ('(a|b)c*d', 'abcd', SUCCEED, 'found+"-"+g1', 'bcd-b'), ('(ab|ab*)bc', 'abc', SUCCEED, 'found+"-"+g1', 'abc-a'), ('a([bc]*)c*', 'abc', SUCCEED, 'found+"-"+g1', 'abc-bc'), ('a([bc]*)(c*d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-bc-d'), ('a([bc]+)(c*d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-bc-d'), ('a([bc]*)(c+d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-b-cd'), ('a[bcd]*dcdcde', 'adcdcde', SUCCEED, 'found', 'adcdcde'), ('a[bcd]+dcdcde', 'adcdcde', FAIL), ('(ab|a)b*c', 'abc', SUCCEED, 'found+"-"+g1', 'abc-ab'), ('((a)(b)c)(d)', 'abcd', SUCCEED, 'g1+"-"+g2+"-"+g3+"-"+g4', 'abc-a-b-d'), ('[a-zA-Z_][a-zA-Z0-9_]*', 'alpha', SUCCEED, 'found', 'alpha'), ('^a(bc+|b[eh])g|.h$', 'abh', SUCCEED, 'found+"-"+g1', 'bh-None'), ('(bc+d$|ef*g.|h?i(j|k))', 'effgz', SUCCEED, 'found+"-"+g1+"-"+g2', 'effgz-effgz-None'), ('(bc+d$|ef*g.|h?i(j|k))', 'ij', SUCCEED, 'found+"-"+g1+"-"+g2', 'ij-ij-j'), ('(bc+d$|ef*g.|h?i(j|k))', 'effg', FAIL), ('(bc+d$|ef*g.|h?i(j|k))', 'bcdd', FAIL), ('(bc+d$|ef*g.|h?i(j|k))', 'reffgz', SUCCEED, 'found+"-"+g1+"-"+g2', 'effgz-effgz-None'), ('(((((((((a)))))))))', 'a', SUCCEED, 'found', 'a'), ('multiple words of text', 'uh-uh', FAIL), ('multiple words', 'multiple words, yeah', SUCCEED, 'found', 'multiple words'), ('(.*)c(.*)', 'abcde', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcde-ab-de'), ('\\((.*), (.*)\\)', '(a, b)', SUCCEED, 'g2+"-"+g1', 'b-a'), ('[k]', 'ab', FAIL), ('a[-]?c', 'ac', SUCCEED, 'found', 'ac'), ('(abc)\\1', 'abcabc', SUCCEED, 'g1', 'abc'), ('([a-c]*)\\1', 'abcabc', SUCCEED, 'g1', 'abc'), ('^(.+)?B', 'AB', SUCCEED, 'g1', 'A'), ('(a+).\\1$', 'aaaaa', SUCCEED, 'found+"-"+g1', 'aaaaa-aa'), ('^(a+).\\1$', 'aaaa', FAIL), ('(abc)\\1', 'abcabc', SUCCEED, 'found+"-"+g1', 'abcabc-abc'), ('([a-c]+)\\1', 'abcabc', SUCCEED, 'found+"-"+g1', 'abcabc-abc'), ('(a)\\1', 'aa', SUCCEED, 'found+"-"+g1', 'aa-a'), ('(a+)\\1', 'aa', SUCCEED, 'found+"-"+g1', 'aa-a'), ('(a+)+\\1', 'aa', SUCCEED, 'found+"-"+g1', 'aa-a'), ('(a).+\\1', 'aba', SUCCEED, 'found+"-"+g1', 'aba-a'), ('(a)ba*\\1', 'aba', SUCCEED, 'found+"-"+g1', 'aba-a'), ('(aa|a)a\\1$', 'aaa', SUCCEED, 'found+"-"+g1', 'aaa-a'), ('(a|aa)a\\1$', 'aaa', SUCCEED, 'found+"-"+g1', 'aaa-a'), ('(a+)a\\1$', 'aaa', SUCCEED, 'found+"-"+g1', 'aaa-a'), ('([abc]*)\\1', 'abcabc', SUCCEED, 'found+"-"+g1', 'abcabc-abc'), ('(a)(b)c|ab', 'ab', SUCCEED, 'found+"-"+g1+"-"+g2', 'ab-None-None'), ('(a)+x', 'aaax', SUCCEED, 'found+"-"+g1', 'aaax-a'), ('([ac])+x', 'aacx', SUCCEED, 'found+"-"+g1', 'aacx-c'), ('([^/]*/)*sub1/', 'd:msgs/tdir/sub1/trial/away.cpp', SUCCEED, 'found+"-"+g1', 'd:msgs/tdir/sub1/-tdir/'), ('([^.]*)\\.([^:]*):[T ]+(.*)', 'track1.title:TBlah blah blah', SUCCEED, 'found+"-"+g1+"-"+g2+"-"+g3', 'track1.title:TBlah blah blah-track1-title-Blah blah blah'), ('([^N]*N)+', 'abNNxyzN', SUCCEED, 'found+"-"+g1', 'abNNxyzN-xyzN'), ('([^N]*N)+', 'abNNxyz', SUCCEED, 'found+"-"+g1', 'abNN-N'), ('([abc]*)x', 'abcx', SUCCEED, 'found+"-"+g1', 'abcx-abc'), ('([abc]*)x', 'abc', FAIL), ('([xyz]*)x', 'abcx', SUCCEED, 'found+"-"+g1', 'x-'), ('(a)+b|aac', 'aac', SUCCEED, 'found+"-"+g1', 'aac-None'), # Test symbolic groups ('(?Paaa)a', 'aaaa', SYNTAX_ERROR), ('(?Paaa)a', 'aaaa', SUCCEED, 'found+"-"+id', 'aaaa-aaa'), ('(?Paa)(?P=id)', 'aaaa', SUCCEED, 'found+"-"+id', 'aaaa-aa'), ('(?Paa)(?P=xd)', 'aaaa', SYNTAX_ERROR), # Test octal escapes/memory references ('\\1', 'a', SYNTAX_ERROR), ('\\09', chr(0) + '9', SUCCEED, 'found', chr(0) + '9'), ('\\141', 'a', SUCCEED, 'found', 'a'), ('(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)\\119', 'abcdefghijklk9', SUCCEED, 'found+"-"+g11', 'abcdefghijklk9-k'), # All tests from Perl ('abc', 'abc', SUCCEED, 'found', 'abc'), ('abc', 'xbc', FAIL), ('abc', 'axc', FAIL), ('abc', 'abx', FAIL), ('abc', 'xabcy', SUCCEED, 'found', 'abc'), ('abc', 'ababc', SUCCEED, 'found', 'abc'), ('ab*c', 'abc', SUCCEED, 'found', 'abc'), ('ab*bc', 'abc', SUCCEED, 'found', 'abc'), ('ab*bc', 'abbc', SUCCEED, 'found', 'abbc'), ('ab*bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'), ('ab{0,}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'), ('ab+bc', 'abbc', SUCCEED, 'found', 'abbc'), ('ab+bc', 'abc', FAIL), ('ab+bc', 'abq', FAIL), ('ab{1,}bc', 'abq', FAIL), ('ab+bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'), ('ab{1,}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'), ('ab{1,3}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'), ('ab{3,4}bc', 'abbbbc', SUCCEED, 'found', 'abbbbc'), ('ab{4,5}bc', 'abbbbc', FAIL), ('ab?bc', 'abbc', SUCCEED, 'found', 'abbc'), ('ab?bc', 'abc', SUCCEED, 'found', 'abc'), ('ab{0,1}bc', 'abc', SUCCEED, 'found', 'abc'), ('ab?bc', 'abbbbc', FAIL), ('ab?c', 'abc', SUCCEED, 'found', 'abc'), ('ab{0,1}c', 'abc', SUCCEED, 'found', 'abc'), ('^abc$', 'abc', SUCCEED, 'found', 'abc'), ('^abc$', 'abcc', FAIL), ('^abc', 'abcc', SUCCEED, 'found', 'abc'), ('^abc$', 'aabc', FAIL), ('abc$', 'aabc', SUCCEED, 'found', 'abc'), ('^', 'abc', SUCCEED, 'found', ''), ('$', 'abc', SUCCEED, 'found', ''), ('a.c', 'abc', SUCCEED, 'found', 'abc'), ('a.c', 'axc', SUCCEED, 'found', 'axc'), ('a.*c', 'axyzc', SUCCEED, 'found', 'axyzc'), ('a.*c', 'axyzd', FAIL), ('a[bc]d', 'abc', FAIL), ('a[bc]d', 'abd', SUCCEED, 'found', 'abd'), ('a[b-d]e', 'abd', FAIL), ('a[b-d]e', 'ace', SUCCEED, 'found', 'ace'), ('a[b-d]', 'aac', SUCCEED, 'found', 'ac'), ('a[-b]', 'a-', SUCCEED, 'found', 'a-'), ('a[b-]', 'a-', SUCCEED, 'found', 'a-'), ('a[b-a]', '-', SYNTAX_ERROR), ('a[]b', '-', SYNTAX_ERROR), ('a[', '-', SYNTAX_ERROR), ('a]', 'a]', SUCCEED, 'found', 'a]'), ('a[]]b', 'a]b', SUCCEED, 'found', 'a]b'), ('a[^bc]d', 'aed', SUCCEED, 'found', 'aed'), ('a[^bc]d', 'abd', FAIL), ('a[^-b]c', 'adc', SUCCEED, 'found', 'adc'), ('a[^-b]c', 'a-c', FAIL), ('a[^]b]c', 'a]c', FAIL), ('a[^]b]c', 'adc', SUCCEED, 'found', 'adc'), ('ab|cd', 'abc', SUCCEED, 'found', 'ab'), ('ab|cd', 'abcd', SUCCEED, 'found', 'ab'), ('()ef', 'def', SUCCEED, 'found+"-"+g1', 'ef-'), ('*a', '-', SYNTAX_ERROR), ('(*)b', '-', SYNTAX_ERROR), ('$b', 'b', FAIL), ('a\\', '-', SYNTAX_ERROR), ('a\\(b', 'a(b', SUCCEED, 'found+"-"+g1', 'a(b-Error'), ('a\\(*b', 'ab', SUCCEED, 'found', 'ab'), ('a\\(*b', 'a((b', SUCCEED, 'found', 'a((b'), ('a\\\\b', 'a\\b', SUCCEED, 'found', 'a\\b'), ('abc)', '-', SYNTAX_ERROR), ('(abc', '-', SYNTAX_ERROR), ('((a))', 'abc', SUCCEED, 'found+"-"+g1+"-"+g2', 'a-a-a'), ('(a)b(c)', 'abc', SUCCEED, 'found+"-"+g1+"-"+g2', 'abc-a-c'), ('a+b+c', 'aabbabc', SUCCEED, 'found', 'abc'), ('a{1,}b{1,}c', 'aabbabc', SUCCEED, 'found', 'abc'), ('a**', '-', SYNTAX_ERROR), ('a.+?c', 'abcabc', SUCCEED, 'found', 'abc'), ('(a+|b)*', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'), ('(a+|b){0,}', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'), ('(a+|b)+', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'), ('(a+|b){1,}', 'ab', SUCCEED, 'found+"-"+g1', 'ab-b'), ('(a+|b)?', 'ab', SUCCEED, 'found+"-"+g1', 'a-a'), ('(a+|b){0,1}', 'ab', SUCCEED, 'found+"-"+g1', 'a-a'), (')(', '-', SYNTAX_ERROR), ('[^ab]*', 'cde', SUCCEED, 'found', 'cde'), ('abc', '', FAIL), ('a*', '', SUCCEED, 'found', ''), ('([abc])*d', 'abbbcd', SUCCEED, 'found+"-"+g1', 'abbbcd-c'), ('([abc])*bcd', 'abcd', SUCCEED, 'found+"-"+g1', 'abcd-a'), ('a|b|c|d|e', 'e', SUCCEED, 'found', 'e'), ('(a|b|c|d|e)f', 'ef', SUCCEED, 'found+"-"+g1', 'ef-e'), ('abcd*efg', 'abcdefg', SUCCEED, 'found', 'abcdefg'), ('ab*', 'xabyabbbz', SUCCEED, 'found', 'ab'), ('ab*', 'xayabbbz', SUCCEED, 'found', 'a'), ('(ab|cd)e', 'abcde', SUCCEED, 'found+"-"+g1', 'cde-cd'), ('[abhgefdc]ij', 'hij', SUCCEED, 'found', 'hij'), ('^(ab|cd)e', 'abcde', FAIL), ('(abc|)ef', 'abcdef', SUCCEED, 'found+"-"+g1', 'ef-'), ('(a|b)c*d', 'abcd', SUCCEED, 'found+"-"+g1', 'bcd-b'), ('(ab|ab*)bc', 'abc', SUCCEED, 'found+"-"+g1', 'abc-a'), ('a([bc]*)c*', 'abc', SUCCEED, 'found+"-"+g1', 'abc-bc'), ('a([bc]*)(c*d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-bc-d'), ('a([bc]+)(c*d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-bc-d'), ('a([bc]*)(c+d)', 'abcd', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcd-b-cd'), ('a[bcd]*dcdcde', 'adcdcde', SUCCEED, 'found', 'adcdcde'), ('a[bcd]+dcdcde', 'adcdcde', FAIL), ('(ab|a)b*c', 'abc', SUCCEED, 'found+"-"+g1', 'abc-ab'), ('((a)(b)c)(d)', 'abcd', SUCCEED, 'g1+"-"+g2+"-"+g3+"-"+g4', 'abc-a-b-d'), ('[a-zA-Z_][a-zA-Z0-9_]*', 'alpha', SUCCEED, 'found', 'alpha'), ('^a(bc+|b[eh])g|.h$', 'abh', SUCCEED, 'found+"-"+g1', 'bh-None'), ('(bc+d$|ef*g.|h?i(j|k))', 'effgz', SUCCEED, 'found+"-"+g1+"-"+g2', 'effgz-effgz-None'), ('(bc+d$|ef*g.|h?i(j|k))', 'ij', SUCCEED, 'found+"-"+g1+"-"+g2', 'ij-ij-j'), ('(bc+d$|ef*g.|h?i(j|k))', 'effg', FAIL), ('(bc+d$|ef*g.|h?i(j|k))', 'bcdd', FAIL), ('(bc+d$|ef*g.|h?i(j|k))', 'reffgz', SUCCEED, 'found+"-"+g1+"-"+g2', 'effgz-effgz-None'), ('((((((((((a))))))))))', 'a', SUCCEED, 'g10', 'a'), ('((((((((((a))))))))))\\10', 'aa', SUCCEED, 'found', 'aa'), # Python does not have the same rules for \\41 so this is a syntax error # ('((((((((((a))))))))))\\41', 'aa', FAIL), # ('((((((((((a))))))))))\\41', 'a!', SUCCEED, 'found', 'a!'), ('((((((((((a))))))))))\\41', '', SYNTAX_ERROR), ('(?i)((((((((((a))))))))))\\41', '', SYNTAX_ERROR), ('(((((((((a)))))))))', 'a', SUCCEED, 'found', 'a'), ('multiple words of text', 'uh-uh', FAIL), ('multiple words', 'multiple words, yeah', SUCCEED, 'found', 'multiple words'), ('(.*)c(.*)', 'abcde', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcde-ab-de'), ('\\((.*), (.*)\\)', '(a, b)', SUCCEED, 'g2+"-"+g1', 'b-a'), ('[k]', 'ab', FAIL), ('a[-]?c', 'ac', SUCCEED, 'found', 'ac'), ('(abc)\\1', 'abcabc', SUCCEED, 'g1', 'abc'), ('([a-c]*)\\1', 'abcabc', SUCCEED, 'g1', 'abc'), ('(?i)abc', 'ABC', SUCCEED, 'found', 'ABC'), ('(?i)abc', 'XBC', FAIL), ('(?i)abc', 'AXC', FAIL), ('(?i)abc', 'ABX', FAIL), ('(?i)abc', 'XABCY', SUCCEED, 'found', 'ABC'), ('(?i)abc', 'ABABC', SUCCEED, 'found', 'ABC'), ('(?i)ab*c', 'ABC', SUCCEED, 'found', 'ABC'), ('(?i)ab*bc', 'ABC', SUCCEED, 'found', 'ABC'), ('(?i)ab*bc', 'ABBC', SUCCEED, 'found', 'ABBC'), ('(?i)ab*?bc', 'ABBBBC', SUCCEED, 'found', 'ABBBBC'), ('(?i)ab{0,}?bc', 'ABBBBC', SUCCEED, 'found', 'ABBBBC'), ('(?i)ab+?bc', 'ABBC', SUCCEED, 'found', 'ABBC'), ('(?i)ab+bc', 'ABC', FAIL), ('(?i)ab+bc', 'ABQ', FAIL), ('(?i)ab{1,}bc', 'ABQ', FAIL), ('(?i)ab+bc', 'ABBBBC', SUCCEED, 'found', 'ABBBBC'), ('(?i)ab{1,}?bc', 'ABBBBC', SUCCEED, 'found', 'ABBBBC'), ('(?i)ab{1,3}?bc', 'ABBBBC', SUCCEED, 'found', 'ABBBBC'), ('(?i)ab{3,4}?bc', 'ABBBBC', SUCCEED, 'found', 'ABBBBC'), ('(?i)ab{4,5}?bc', 'ABBBBC', FAIL), ('(?i)ab??bc', 'ABBC', SUCCEED, 'found', 'ABBC'), ('(?i)ab??bc', 'ABC', SUCCEED, 'found', 'ABC'), ('(?i)ab{0,1}?bc', 'ABC', SUCCEED, 'found', 'ABC'), ('(?i)ab??bc', 'ABBBBC', FAIL), ('(?i)ab??c', 'ABC', SUCCEED, 'found', 'ABC'), ('(?i)ab{0,1}?c', 'ABC', SUCCEED, 'found', 'ABC'), ('(?i)^abc$', 'ABC', SUCCEED, 'found', 'ABC'), ('(?i)^abc$', 'ABCC', FAIL), ('(?i)^abc', 'ABCC', SUCCEED, 'found', 'ABC'), ('(?i)^abc$', 'AABC', FAIL), ('(?i)abc$', 'AABC', SUCCEED, 'found', 'ABC'), ('(?i)^', 'ABC', SUCCEED, 'found', ''), ('(?i)$', 'ABC', SUCCEED, 'found', ''), ('(?i)a.c', 'ABC', SUCCEED, 'found', 'ABC'), ('(?i)a.c', 'AXC', SUCCEED, 'found', 'AXC'), ('(?i)a.*?c', 'AXYZC', SUCCEED, 'found', 'AXYZC'), ('(?i)a.*c', 'AXYZD', FAIL), ('(?i)a[bc]d', 'ABC', FAIL), ('(?i)a[bc]d', 'ABD', SUCCEED, 'found', 'ABD'), ('(?i)a[b-d]e', 'ABD', FAIL), ('(?i)a[b-d]e', 'ACE', SUCCEED, 'found', 'ACE'), ('(?i)a[b-d]', 'AAC', SUCCEED, 'found', 'AC'), ('(?i)a[-b]', 'A-', SUCCEED, 'found', 'A-'), ('(?i)a[b-]', 'A-', SUCCEED, 'found', 'A-'), ('(?i)a[b-a]', '-', SYNTAX_ERROR), ('(?i)a[]b', '-', SYNTAX_ERROR), ('(?i)a[', '-', SYNTAX_ERROR), ('(?i)a]', 'A]', SUCCEED, 'found', 'A]'), ('(?i)a[]]b', 'A]B', SUCCEED, 'found', 'A]B'), ('(?i)a[^bc]d', 'AED', SUCCEED, 'found', 'AED'), ('(?i)a[^bc]d', 'ABD', FAIL), ('(?i)a[^-b]c', 'ADC', SUCCEED, 'found', 'ADC'), ('(?i)a[^-b]c', 'A-C', FAIL), ('(?i)a[^]b]c', 'A]C', FAIL), ('(?i)a[^]b]c', 'ADC', SUCCEED, 'found', 'ADC'), ('(?i)ab|cd', 'ABC', SUCCEED, 'found', 'AB'), ('(?i)ab|cd', 'ABCD', SUCCEED, 'found', 'AB'), ('(?i)()ef', 'DEF', SUCCEED, 'found+"-"+g1', 'EF-'), ('(?i)*a', '-', SYNTAX_ERROR), ('(?i)(*)b', '-', SYNTAX_ERROR), ('(?i)$b', 'B', FAIL), ('(?i)a\\', '-', SYNTAX_ERROR), ('(?i)a\\(b', 'A(B', SUCCEED, 'found+"-"+g1', 'A(B-Error'), ('(?i)a\\(*b', 'AB', SUCCEED, 'found', 'AB'), ('(?i)a\\(*b', 'A((B', SUCCEED, 'found', 'A((B'), ('(?i)a\\\\b', 'A\\B', SUCCEED, 'found', 'A\\B'), ('(?i)abc)', '-', SYNTAX_ERROR), ('(?i)(abc', '-', SYNTAX_ERROR), ('(?i)((a))', 'ABC', SUCCEED, 'found+"-"+g1+"-"+g2', 'A-A-A'), ('(?i)(a)b(c)', 'ABC', SUCCEED, 'found+"-"+g1+"-"+g2', 'ABC-A-C'), ('(?i)a+b+c', 'AABBABC', SUCCEED, 'found', 'ABC'), ('(?i)a{1,}b{1,}c', 'AABBABC', SUCCEED, 'found', 'ABC'), ('(?i)a**', '-', SYNTAX_ERROR), ('(?i)a.+?c', 'ABCABC', SUCCEED, 'found', 'ABC'), ('(?i)a.*?c', 'ABCABC', SUCCEED, 'found', 'ABC'), ('(?i)a.{0,5}?c', 'ABCABC', SUCCEED, 'found', 'ABC'), ('(?i)(a+|b)*', 'AB', SUCCEED, 'found+"-"+g1', 'AB-B'), ('(?i)(a+|b){0,}', 'AB', SUCCEED, 'found+"-"+g1', 'AB-B'), ('(?i)(a+|b)+', 'AB', SUCCEED, 'found+"-"+g1', 'AB-B'), ('(?i)(a+|b){1,}', 'AB', SUCCEED, 'found+"-"+g1', 'AB-B'), ('(?i)(a+|b)?', 'AB', SUCCEED, 'found+"-"+g1', 'A-A'), ('(?i)(a+|b){0,1}', 'AB', SUCCEED, 'found+"-"+g1', 'A-A'), ('(?i)(a+|b){0,1}?', 'AB', SUCCEED, 'found+"-"+g1', '-None'), ('(?i))(', '-', SYNTAX_ERROR), ('(?i)[^ab]*', 'CDE', SUCCEED, 'found', 'CDE'), ('(?i)abc', '', FAIL), ('(?i)a*', '', SUCCEED, 'found', ''), ('(?i)([abc])*d', 'ABBBCD', SUCCEED, 'found+"-"+g1', 'ABBBCD-C'), ('(?i)([abc])*bcd', 'ABCD', SUCCEED, 'found+"-"+g1', 'ABCD-A'), ('(?i)a|b|c|d|e', 'E', SUCCEED, 'found', 'E'), ('(?i)(a|b|c|d|e)f', 'EF', SUCCEED, 'found+"-"+g1', 'EF-E'), ('(?i)abcd*efg', 'ABCDEFG', SUCCEED, 'found', 'ABCDEFG'), ('(?i)ab*', 'XABYABBBZ', SUCCEED, 'found', 'AB'), ('(?i)ab*', 'XAYABBBZ', SUCCEED, 'found', 'A'), ('(?i)(ab|cd)e', 'ABCDE', SUCCEED, 'found+"-"+g1', 'CDE-CD'), ('(?i)[abhgefdc]ij', 'HIJ', SUCCEED, 'found', 'HIJ'), ('(?i)^(ab|cd)e', 'ABCDE', FAIL), ('(?i)(abc|)ef', 'ABCDEF', SUCCEED, 'found+"-"+g1', 'EF-'), ('(?i)(a|b)c*d', 'ABCD', SUCCEED, 'found+"-"+g1', 'BCD-B'), ('(?i)(ab|ab*)bc', 'ABC', SUCCEED, 'found+"-"+g1', 'ABC-A'), ('(?i)a([bc]*)c*', 'ABC', SUCCEED, 'found+"-"+g1', 'ABC-BC'), ('(?i)a([bc]*)(c*d)', 'ABCD', SUCCEED, 'found+"-"+g1+"-"+g2', 'ABCD-BC-D'), ('(?i)a([bc]+)(c*d)', 'ABCD', SUCCEED, 'found+"-"+g1+"-"+g2', 'ABCD-BC-D'), ('(?i)a([bc]*)(c+d)', 'ABCD', SUCCEED, 'found+"-"+g1+"-"+g2', 'ABCD-B-CD'), ('(?i)a[bcd]*dcdcde', 'ADCDCDE', SUCCEED, 'found', 'ADCDCDE'), ('(?i)a[bcd]+dcdcde', 'ADCDCDE', FAIL), ('(?i)(ab|a)b*c', 'ABC', SUCCEED, 'found+"-"+g1', 'ABC-AB'), ('(?i)((a)(b)c)(d)', 'ABCD', SUCCEED, 'g1+"-"+g2+"-"+g3+"-"+g4', 'ABC-A-B-D'), ('(?i)[a-zA-Z_][a-zA-Z0-9_]*', 'ALPHA', SUCCEED, 'found', 'ALPHA'), ('(?i)^a(bc+|b[eh])g|.h$', 'ABH', SUCCEED, 'found+"-"+g1', 'BH-None'), ('(?i)(bc+d$|ef*g.|h?i(j|k))', 'EFFGZ', SUCCEED, 'found+"-"+g1+"-"+g2', 'EFFGZ-EFFGZ-None'), ('(?i)(bc+d$|ef*g.|h?i(j|k))', 'IJ', SUCCEED, 'found+"-"+g1+"-"+g2', 'IJ-IJ-J'), ('(?i)(bc+d$|ef*g.|h?i(j|k))', 'EFFG', FAIL), ('(?i)(bc+d$|ef*g.|h?i(j|k))', 'BCDD', FAIL), ('(?i)(bc+d$|ef*g.|h?i(j|k))', 'REFFGZ', SUCCEED, 'found+"-"+g1+"-"+g2', 'EFFGZ-EFFGZ-None'), ('(?i)((((((((((a))))))))))', 'A', SUCCEED, 'g10', 'A'), ('(?i)((((((((((a))))))))))\\10', 'AA', SUCCEED, 'found', 'AA'), #('(?i)((((((((((a))))))))))\\41', 'AA', FAIL), #('(?i)((((((((((a))))))))))\\41', 'A!', SUCCEED, 'found', 'A!'), ('(?i)(((((((((a)))))))))', 'A', SUCCEED, 'found', 'A'), ('(?i)(?:(?:(?:(?:(?:(?:(?:(?:(?:(a))))))))))', 'A', SUCCEED, 'g1', 'A'), ('(?i)(?:(?:(?:(?:(?:(?:(?:(?:(?:(a|b|c))))))))))', 'C', SUCCEED, 'g1', 'C'), ('(?i)multiple words of text', 'UH-UH', FAIL), ('(?i)multiple words', 'MULTIPLE WORDS, YEAH', SUCCEED, 'found', 'MULTIPLE WORDS'), ('(?i)(.*)c(.*)', 'ABCDE', SUCCEED, 'found+"-"+g1+"-"+g2', 'ABCDE-AB-DE'), ('(?i)\\((.*), (.*)\\)', '(A, B)', SUCCEED, 'g2+"-"+g1', 'B-A'), ('(?i)[k]', 'AB', FAIL), # ('(?i)abcd', 'ABCD', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'ABCD-$&-\\ABCD'), # ('(?i)a(bc)d', 'ABCD', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'BC-$1-\\BC'), ('(?i)a[-]?c', 'AC', SUCCEED, 'found', 'AC'), ('(?i)(abc)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'), ('(?i)([a-c]*)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'), ('a(?!b).', 'abad', SUCCEED, 'found', 'ad'), ('a(?=d).', 'abad', SUCCEED, 'found', 'ad'), ('a(?=c|d).', 'abad', SUCCEED, 'found', 'ad'), ('a(?:b|c|d)(.)', 'ace', SUCCEED, 'g1', 'e'), ('a(?:b|c|d)*(.)', 'ace', SUCCEED, 'g1', 'e'), ('a(?:b|c|d)+?(.)', 'ace', SUCCEED, 'g1', 'e'), ('a(?:b|(c|e){1,2}?|d)+?(.)', 'ace', SUCCEED, 'g1 + g2', 'ce'), ('^(.+)?B', 'AB', SUCCEED, 'g1', 'A'), # lookbehind: split by : but not if it is escaped by -. ('(?]*?b', 'a>b', FAIL), # bug 490573: minimizing repeat problem (r'^a*?$', 'foo', FAIL), # bug 470582: nested groups problem (r'^((a)c)?(ab)$', 'ab', SUCCEED, 'g1+"-"+g2+"-"+g3', 'None-None-ab'), # another minimizing repeat problem (capturing groups in assertions) ('^([ab]*?)(?=(b)?)c', 'abc', SUCCEED, 'g1+"-"+g2', 'ab-None'), ('^([ab]*?)(?!(b))c', 'abc', SUCCEED, 'g1+"-"+g2', 'ab-None'), ('^([ab]*?)(? self.maxother: i = max(0, (self.maxother-3)//2) j = max(0, self.maxother-3-i) s = s[:i] + '...' + s[len(s)-j:] return s def _repr_iterable(self, x, level, left, right, maxiter, trail=''): n = len(x) if level <= 0 and n: s = '...' else: newlevel = level - 1 repr1 = self.repr1 pieces = [repr1(elem, newlevel) for elem in itertools.islice(x, maxiter)] if n > maxiter: pieces.append('...') s = ', '.join(pieces) if n == 1 and trail: right = trail + right return '%s%s%s' % (left, s, right) def repr_tuple(self, x, level): return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',') def repr_list(self, x, level): return self._repr_iterable(x, level, '[', ']', self.maxlist) def repr_array(self, x, level): header = "array('%s', [" % x.typecode return self._repr_iterable(x, level, header, '])', self.maxarray) def repr_set(self, x, level): x = _possibly_sorted(x) return self._repr_iterable(x, level, 'set([', '])', self.maxset) def repr_frozenset(self, x, level): x = _possibly_sorted(x) return self._repr_iterable(x, level, 'frozenset([', '])', self.maxfrozenset) def repr_deque(self, x, level): return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque) def repr_dict(self, x, level): n = len(x) if n == 0: return '{}' if level <= 0: return '{...}' newlevel = level - 1 repr1 = self.repr1 pieces = [] for key in itertools.islice(_possibly_sorted(x), self.maxdict): keyrepr = repr1(key, newlevel) valrepr = repr1(x[key], newlevel) pieces.append('%s: %s' % (keyrepr, valrepr)) if n > self.maxdict: pieces.append('...') s = ', '.join(pieces) return '{%s}' % (s,) def repr_str(self, x, level): s = __builtin__.repr(x[:self.maxstring]) if len(s) > self.maxstring: i = max(0, (self.maxstring-3)//2) j = max(0, self.maxstring-3-i) s = __builtin__.repr(x[:i] + x[len(x)-j:]) s = s[:i] + '...' + s[len(s)-j:] return s def repr_long(self, x, level): s = __builtin__.repr(x) # XXX Hope this isn't too slow... if len(s) > self.maxlong: i = max(0, (self.maxlong-3)//2) j = max(0, self.maxlong-3-i) s = s[:i] + '...' + s[len(s)-j:] return s def repr_instance(self, x, level): try: s = __builtin__.repr(x) # Bugs in x.__repr__() can cause arbitrary # exceptions -- then make up something except Exception: return '<%s instance at %x>' % (x.__class__.__name__, id(x)) if len(s) > self.maxstring: i = max(0, (self.maxstring-3)//2) j = max(0, self.maxstring-3-i) s = s[:i] + '...' + s[len(s)-j:] return s def _possibly_sorted(x): # Since not all sequences of items can be sorted and comparison # functions may raise arbitrary exceptions, return an unsorted # sequence in that case. try: return sorted(x) except Exception: return list(x) aRepr = Repr() repr = aRepr.repr ================================================ FILE: third_party/stdlib/rfc822.py ================================================ """RFC 2822 message manipulation. Note: This is only a very rough sketch of a full RFC-822 parser; in particular the tokenizing of addresses does not adhere to all the quoting rules. Note: RFC 2822 is a long awaited update to RFC 822. This module should conform to RFC 2822, and is thus mis-named (it's not worth renaming it). Some effort at RFC 2822 updates have been made, but a thorough audit has not been performed. Consider any RFC 2822 non-conformance to be a bug. RFC 2822: http://www.faqs.org/rfcs/rfc2822.html RFC 822 : http://www.faqs.org/rfcs/rfc822.html (obsolete) Directions for use: To create a Message object: first open a file, e.g.: fp = open(file, 'r') You can use any other legal way of getting an open file object, e.g. use sys.stdin or call os.popen(). Then pass the open file object to the Message() constructor: m = Message(fp) This class can work with any input object that supports a readline method. If the input object has seek and tell capability, the rewindbody method will work; also illegal lines will be pushed back onto the input stream. If the input object lacks seek but has an `unread' method that can push back a line of input, Message will use that to push back illegal lines. Thus this class can be used to parse messages coming from a buffered stream. The optional `seekable' argument is provided as a workaround for certain stdio libraries in which tell() discards buffered data before discovering that the lseek() system call doesn't work. For maximum portability, you should set the seekable argument to zero to prevent that initial \code{tell} when passing in an unseekable object such as a file object created from a socket object. If it is 1 on entry -- which it is by default -- the tell() method of the open file object is called once; if this raises an exception, seekable is reset to 0. For other nonzero values of seekable, this test is not made. To get the text of a particular header there are several methods: str = m.getheader(name) str = m.getrawheader(name) where name is the name of the header, e.g. 'Subject'. The difference is that getheader() strips the leading and trailing whitespace, while getrawheader() doesn't. Both functions retain embedded whitespace (including newlines) exactly as they are specified in the header, and leave the case of the text unchanged. For addresses and address lists there are functions realname, mailaddress = m.getaddr(name) list = m.getaddrlist(name) where the latter returns a list of (realname, mailaddr) tuples. There is also a method time = m.getdate(name) which parses a Date-like field and returns a time-compatible tuple, i.e. a tuple such as returned by time.localtime() or accepted by time.mktime(). See the class definition for lower level access methods. There are also some utility functions here. """ # Cleanup and extensions by Eric S. Raymond import time from warnings import warnpy3k warnpy3k("in 3.x, rfc822 has been removed in favor of the email package", stacklevel=2) __all__ = ["Message","AddressList","parsedate","parsedate_tz","mktime_tz"] _blanklines = ('\r\n', '\n') # Optimization for islast() class Message(object): """Represents a single RFC 2822-compliant message.""" def __init__(self, fp, seekable = 1): """Initialize the class instance and read the headers.""" if seekable == 1: # Exercise tell() to make sure it works # (and then assume seek() works, too) try: fp.tell() except (AttributeError, IOError): seekable = 0 self.fp = fp self.seekable = seekable self.startofheaders = None self.startofbody = None # if self.seekable: try: self.startofheaders = self.fp.tell() except IOError: self.seekable = 0 # self.readheaders() # if self.seekable: try: self.startofbody = self.fp.tell() except IOError: self.seekable = 0 def rewindbody(self): """Rewind the file to the start of the body (if seekable).""" if not self.seekable: raise IOError, "unseekable file" self.fp.seek(self.startofbody) def readheaders(self): """Read header lines. Read header lines up to the entirely blank line that terminates them. The (normally blank) line that ends the headers is skipped, but not included in the returned list. If a non-header line ends the headers, (which is an error), an attempt is made to backspace over it; it is never included in the returned list. The variable self.status is set to the empty string if all went well, otherwise it is an error message. The variable self.headers is a completely uninterpreted list of lines contained in the header (so printing them will reproduce the header exactly as it appears in the file). """ self.dict = {} self.unixfrom = '' self.headers = lst = [] self.status = '' headerseen = "" firstline = 1 startofline = unread = tell = None if hasattr(self.fp, 'unread'): unread = self.fp.unread elif self.seekable: tell = self.fp.tell while 1: if tell: try: startofline = tell() except IOError: startofline = tell = None self.seekable = 0 line = self.fp.readline() if not line: self.status = 'EOF in headers' break # Skip unix From name time lines if firstline and line.startswith('From '): self.unixfrom = self.unixfrom + line continue firstline = 0 if headerseen and line[0] in ' \t': # It's a continuation line. lst.append(line) x = (self.dict[headerseen] + "\n " + line.strip()) self.dict[headerseen] = x.strip() continue elif self.iscomment(line): # It's a comment. Ignore it. continue elif self.islast(line): # Note! No pushback here! The delimiter line gets eaten. break headerseen = self.isheader(line) if headerseen: # It's a legal header line, save it. lst.append(line) self.dict[headerseen] = line[len(headerseen)+1:].strip() continue elif headerseen is not None: # An empty header name. These aren't allowed in HTTP, but it's # probably a benign mistake. Don't add the header, just keep # going. continue else: # It's not a header line; throw it back and stop here. if not self.dict: self.status = 'No headers' else: self.status = 'Non-header line where header expected' # Try to undo the read. if unread: unread(line) elif tell: self.fp.seek(startofline) else: self.status = self.status + '; bad seek' break def isheader(self, line): """Determine whether a given line is a legal header. This method should return the header name, suitably canonicalized. You may override this method in order to use Message parsing on tagged data in RFC 2822-like formats with special header formats. """ i = line.find(':') if i > -1: return line[:i].lower() return None def islast(self, line): """Determine whether a line is a legal end of RFC 2822 headers. You may override this method if your application wants to bend the rules, e.g. to strip trailing whitespace, or to recognize MH template separators ('--------'). For convenience (e.g. for code reading from sockets) a line consisting of \\r\\n also matches. """ return line in _blanklines def iscomment(self, line): """Determine whether a line should be skipped entirely. You may override this method in order to use Message parsing on tagged data in RFC 2822-like formats that support embedded comments or free-text data. """ return False def getallmatchingheaders(self, name): """Find all header lines matching a given header name. Look through the list of headers and find all lines matching a given header name (and their continuation lines). A list of the lines is returned, without interpretation. If the header does not occur, an empty list is returned. If the header occurs multiple times, all occurrences are returned. Case is not important in the header name. """ name = name.lower() + ':' n = len(name) lst = [] hit = 0 for line in self.headers: if line[:n].lower() == name: hit = 1 elif not line[:1].isspace(): hit = 0 if hit: lst.append(line) return lst def getfirstmatchingheader(self, name): """Get the first header line matching name. This is similar to getallmatchingheaders, but it returns only the first matching header (and its continuation lines). """ name = name.lower() + ':' n = len(name) lst = [] hit = 0 for line in self.headers: if hit: if not line[:1].isspace(): break elif line[:n].lower() == name: hit = 1 if hit: lst.append(line) return lst def getrawheader(self, name): """A higher-level interface to getfirstmatchingheader(). Return a string containing the literal text of the header but with the keyword stripped. All leading, trailing and embedded whitespace is kept in the string, however. Return None if the header does not occur. """ lst = self.getfirstmatchingheader(name) if not lst: return None lst[0] = lst[0][len(name) + 1:] return ''.join(lst) def getheader(self, name, default=None): """Get the header value for a name. This is the normal interface: it returns a stripped version of the header value for a given header name, or None if it doesn't exist. This uses the dictionary version which finds the *last* such header. """ return self.dict.get(name.lower(), default) get = getheader def getheaders(self, name): """Get all values for a header. This returns a list of values for headers given more than once; each value in the result list is stripped in the same way as the result of getheader(). If the header is not given, return an empty list. """ result = [] current = '' have_header = 0 for s in self.getallmatchingheaders(name): if s[0].isspace(): if current: current = "%s\n %s" % (current, s.strip()) else: current = s.strip() else: if have_header: result.append(current) current = s[s.find(":") + 1:].strip() have_header = 1 if have_header: result.append(current) return result def getaddr(self, name): """Get a single address from a header, as a tuple. An example return value: ('Guido van Rossum', 'guido@cwi.nl') """ # New, by Ben Escoto alist = self.getaddrlist(name) if alist: return alist[0] else: return (None, None) def getaddrlist(self, name): """Get a list of addresses from a header. Retrieves a list of addresses from a header, where each address is a tuple as returned by getaddr(). Scans all named headers, so it works properly with multiple To: or Cc: headers for example. """ raw = [] for h in self.getallmatchingheaders(name): if h[0] in ' \t': raw.append(h) else: if raw: raw.append(', ') i = h.find(':') if i > 0: addr = h[i+1:] raw.append(addr) alladdrs = ''.join(raw) a = AddressList(alladdrs) return a.addresslist def getdate(self, name): """Retrieve a date field from a header. Retrieves a date field from the named header, returning a tuple compatible with time.mktime(). """ try: data = self[name] except KeyError: return None return parsedate(data) def getdate_tz(self, name): """Retrieve a date field from a header as a 10-tuple. The first 9 elements make up a tuple compatible with time.mktime(), and the 10th is the offset of the poster's time zone from GMT/UTC. """ try: data = self[name] except KeyError: return None return parsedate_tz(data) # Access as a dictionary (only finds *last* header of each type): def __len__(self): """Get the number of headers in a message.""" return len(self.dict) def __getitem__(self, name): """Get a specific header, as from a dictionary.""" return self.dict[name.lower()] def __setitem__(self, name, value): """Set the value of a header. Note: This is not a perfect inversion of __getitem__, because any changed headers get stuck at the end of the raw-headers list rather than where the altered header was. """ del self[name] # Won't fail if it doesn't exist self.dict[name.lower()] = value text = name + ": " + value for line in text.split("\n"): self.headers.append(line + "\n") def __delitem__(self, name): """Delete all occurrences of a specific header, if it is present.""" name = name.lower() if not name in self.dict: return del self.dict[name] name = name + ':' n = len(name) lst = [] hit = 0 for i in range(len(self.headers)): line = self.headers[i] if line[:n].lower() == name: hit = 1 elif not line[:1].isspace(): hit = 0 if hit: lst.append(i) for i in reversed(lst): del self.headers[i] def setdefault(self, name, default=""): lowername = name.lower() if lowername in self.dict: return self.dict[lowername] else: text = name + ": " + default for line in text.split("\n"): self.headers.append(line + "\n") self.dict[lowername] = default return default def has_key(self, name): """Determine whether a message contains the named header.""" return name.lower() in self.dict def __contains__(self, name): """Determine whether a message contains the named header.""" return name.lower() in self.dict def __iter__(self): return iter(self.dict) def keys(self): """Get all of a message's header field names.""" return self.dict.keys() def values(self): """Get all of a message's header field values.""" return self.dict.values() def items(self): """Get all of a message's headers. Returns a list of name, value tuples. """ return self.dict.items() def __str__(self): return ''.join(self.headers) # Utility functions # ----------------- # XXX Should fix unquote() and quote() to be really conformant. # XXX The inverses of the parse functions may also be useful. def unquote(s): """Remove quotes from a string.""" if len(s) > 1: if s.startswith('"') and s.endswith('"'): return s[1:-1].replace('\\\\', '\\').replace('\\"', '"') if s.startswith('<') and s.endswith('>'): return s[1:-1] return s def quote(s): """Add quotes around a string.""" return s.replace('\\', '\\\\').replace('"', '\\"') def parseaddr(address): """Parse an address into a (realname, mailaddr) tuple.""" a = AddressList(address) lst = a.addresslist if not lst: return (None, None) return lst[0] class AddrlistClass(object): """Address parser class by Ben Escoto. To understand what this class does, it helps to have a copy of RFC 2822 in front of you. http://www.faqs.org/rfcs/rfc2822.html Note: this class interface is deprecated and may be removed in the future. Use rfc822.AddressList instead. """ def __init__(self, field): """Initialize a new instance. `field' is an unparsed address header field, containing one or more addresses. """ self.specials = '()<>@,:;.\"[]' self.pos = 0 self.LWS = ' \t' self.CR = '\r\n' self.atomends = self.specials + self.LWS + self.CR # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it # is obsolete syntax. RFC 2822 requires that we recognize obsolete # syntax, so allow dots in phrases. self.phraseends = self.atomends.replace('.', '') self.field = field self.commentlist = [] def gotonext(self): """Parse up to the start of the next address.""" while self.pos < len(self.field): if self.field[self.pos] in self.LWS + '\n\r': self.pos = self.pos + 1 elif self.field[self.pos] == '(': self.commentlist.append(self.getcomment()) else: break def getaddrlist(self): """Parse all addresses. Returns a list containing all of the addresses. """ result = [] ad = self.getaddress() while ad: result += ad ad = self.getaddress() return result def getaddress(self): """Parse the next address.""" self.commentlist = [] self.gotonext() oldpos = self.pos oldcl = self.commentlist plist = self.getphraselist() self.gotonext() returnlist = [] if self.pos >= len(self.field): # Bad email address technically, no domain. if plist: returnlist = [(' '.join(self.commentlist), plist[0])] elif self.field[self.pos] in '.@': # email address is just an addrspec # this isn't very efficient since we start over self.pos = oldpos self.commentlist = oldcl addrspec = self.getaddrspec() returnlist = [(' '.join(self.commentlist), addrspec)] elif self.field[self.pos] == ':': # address is a group returnlist = [] fieldlen = len(self.field) self.pos += 1 while self.pos < len(self.field): self.gotonext() if self.pos < fieldlen and self.field[self.pos] == ';': self.pos += 1 break returnlist = returnlist + self.getaddress() elif self.field[self.pos] == '<': # Address is a phrase then a route addr routeaddr = self.getrouteaddr() if self.commentlist: returnlist = [(' '.join(plist) + ' (' + \ ' '.join(self.commentlist) + ')', routeaddr)] else: returnlist = [(' '.join(plist), routeaddr)] else: if plist: returnlist = [(' '.join(self.commentlist), plist[0])] elif self.field[self.pos] in self.specials: self.pos += 1 self.gotonext() if self.pos < len(self.field) and self.field[self.pos] == ',': self.pos += 1 return returnlist def getrouteaddr(self): """Parse a route address (Return-path value). This method just skips all the route stuff and returns the addrspec. """ if self.field[self.pos] != '<': return expectroute = 0 self.pos += 1 self.gotonext() adlist = "" while self.pos < len(self.field): if expectroute: self.getdomain() expectroute = 0 elif self.field[self.pos] == '>': self.pos += 1 break elif self.field[self.pos] == '@': self.pos += 1 expectroute = 1 elif self.field[self.pos] == ':': self.pos += 1 else: adlist = self.getaddrspec() self.pos += 1 break self.gotonext() return adlist def getaddrspec(self): """Parse an RFC 2822 addr-spec.""" aslist = [] self.gotonext() while self.pos < len(self.field): if self.field[self.pos] == '.': aslist.append('.') self.pos += 1 elif self.field[self.pos] == '"': aslist.append('"%s"' % self.getquote()) elif self.field[self.pos] in self.atomends: break else: aslist.append(self.getatom()) self.gotonext() if self.pos >= len(self.field) or self.field[self.pos] != '@': return ''.join(aslist) aslist.append('@') self.pos += 1 self.gotonext() return ''.join(aslist) + self.getdomain() def getdomain(self): """Get the complete domain name from an address.""" sdlist = [] while self.pos < len(self.field): if self.field[self.pos] in self.LWS: self.pos += 1 elif self.field[self.pos] == '(': self.commentlist.append(self.getcomment()) elif self.field[self.pos] == '[': sdlist.append(self.getdomainliteral()) elif self.field[self.pos] == '.': self.pos += 1 sdlist.append('.') elif self.field[self.pos] in self.atomends: break else: sdlist.append(self.getatom()) return ''.join(sdlist) def getdelimited(self, beginchar, endchars, allowcomments = 1): """Parse a header fragment delimited by special characters. `beginchar' is the start character for the fragment. If self is not looking at an instance of `beginchar' then getdelimited returns the empty string. `endchars' is a sequence of allowable end-delimiting characters. Parsing stops when one of these is encountered. If `allowcomments' is non-zero, embedded RFC 2822 comments are allowed within the parsed fragment. """ if self.field[self.pos] != beginchar: return '' slist = [''] quote = 0 self.pos += 1 while self.pos < len(self.field): if quote == 1: slist.append(self.field[self.pos]) quote = 0 elif self.field[self.pos] in endchars: self.pos += 1 break elif allowcomments and self.field[self.pos] == '(': slist.append(self.getcomment()) continue # have already advanced pos from getcomment elif self.field[self.pos] == '\\': quote = 1 else: slist.append(self.field[self.pos]) self.pos += 1 return ''.join(slist) def getquote(self): """Get a quote-delimited fragment from self's field.""" return self.getdelimited('"', '"\r', 0) def getcomment(self): """Get a parenthesis-delimited fragment from self's field.""" return self.getdelimited('(', ')\r', 1) def getdomainliteral(self): """Parse an RFC 2822 domain-literal.""" return '[%s]' % self.getdelimited('[', ']\r', 0) def getatom(self, atomends=None): """Parse an RFC 2822 atom. Optional atomends specifies a different set of end token delimiters (the default is to use self.atomends). This is used e.g. in getphraselist() since phrase endings must not include the `.' (which is legal in phrases).""" atomlist = [''] if atomends is None: atomends = self.atomends while self.pos < len(self.field): if self.field[self.pos] in atomends: break else: atomlist.append(self.field[self.pos]) self.pos += 1 return ''.join(atomlist) def getphraselist(self): """Parse a sequence of RFC 2822 phrases. A phrase is a sequence of words, which are in turn either RFC 2822 atoms or quoted-strings. Phrases are canonicalized by squeezing all runs of continuous whitespace into one space. """ plist = [] while self.pos < len(self.field): if self.field[self.pos] in self.LWS: self.pos += 1 elif self.field[self.pos] == '"': plist.append(self.getquote()) elif self.field[self.pos] == '(': self.commentlist.append(self.getcomment()) elif self.field[self.pos] in self.phraseends: break else: plist.append(self.getatom(self.phraseends)) return plist class AddressList(AddrlistClass): """An AddressList encapsulates a list of parsed RFC 2822 addresses.""" def __init__(self, field): AddrlistClass.__init__(self, field) if field: self.addresslist = self.getaddrlist() else: self.addresslist = [] def __len__(self): return len(self.addresslist) def __str__(self): return ", ".join(map(dump_address_pair, self.addresslist)) def __add__(self, other): # Set union newaddr = AddressList(None) newaddr.addresslist = self.addresslist[:] for x in other.addresslist: if not x in self.addresslist: newaddr.addresslist.append(x) return newaddr def __iadd__(self, other): # Set union, in-place for x in other.addresslist: if not x in self.addresslist: self.addresslist.append(x) return self def __sub__(self, other): # Set difference newaddr = AddressList(None) for x in self.addresslist: if not x in other.addresslist: newaddr.addresslist.append(x) return newaddr def __isub__(self, other): # Set difference, in-place for x in other.addresslist: if x in self.addresslist: self.addresslist.remove(x) return self def __getitem__(self, index): # Make indexing, slices, and 'in' work return self.addresslist[index] def dump_address_pair(pair): """Dump a (name, address) pair in a canonicalized form.""" if pair[0]: return '"' + pair[0] + '" <' + pair[1] + '>' else: return pair[1] # Parse a date field _monthnames = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec', 'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december'] _daynames = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] # The timezone table does not include the military time zones defined # in RFC822, other than Z. According to RFC1123, the description in # RFC822 gets the signs wrong, so we can't rely on any such time # zones. RFC1123 recommends that numeric timezone indicators be used # instead of timezone names. _timezones = {'UT':0, 'UTC':0, 'GMT':0, 'Z':0, 'AST': -400, 'ADT': -300, # Atlantic (used in Canada) 'EST': -500, 'EDT': -400, # Eastern 'CST': -600, 'CDT': -500, # Central 'MST': -700, 'MDT': -600, # Mountain 'PST': -800, 'PDT': -700 # Pacific } def parsedate_tz(data): """Convert a date string to a time tuple. Accounts for military timezones. """ if not data: return None data = data.split() if data[0][-1] in (',', '.') or data[0].lower() in _daynames: # There's a dayname here. Skip it del data[0] else: # no space after the "weekday,"? i = data[0].rfind(',') if i >= 0: data[0] = data[0][i+1:] if len(data) == 3: # RFC 850 date, deprecated stuff = data[0].split('-') if len(stuff) == 3: data = stuff + data[1:] if len(data) == 4: s = data[3] i = s.find('+') if i > 0: data[3:] = [s[:i], s[i+1:]] else: data.append('') # Dummy tz if len(data) < 5: return None data = data[:5] [dd, mm, yy, tm, tz] = data mm = mm.lower() if not mm in _monthnames: dd, mm = mm, dd.lower() if not mm in _monthnames: return None mm = _monthnames.index(mm)+1 if mm > 12: mm = mm - 12 if dd[-1] == ',': dd = dd[:-1] i = yy.find(':') if i > 0: yy, tm = tm, yy if yy[-1] == ',': yy = yy[:-1] if not yy[0].isdigit(): yy, tz = tz, yy if tm[-1] == ',': tm = tm[:-1] tm = tm.split(':') if len(tm) == 2: [thh, tmm] = tm tss = '0' elif len(tm) == 3: [thh, tmm, tss] = tm else: return None try: yy = int(yy) dd = int(dd) thh = int(thh) tmm = int(tmm) tss = int(tss) except ValueError: return None tzoffset = None tz = tz.upper() if tz in _timezones: tzoffset = _timezones[tz] else: try: tzoffset = int(tz) except ValueError: pass # Convert a timezone offset into seconds ; -0500 -> -18000 if tzoffset: if tzoffset < 0: tzsign = -1 tzoffset = -tzoffset else: tzsign = 1 tzoffset = tzsign * ( (tzoffset//100)*3600 + (tzoffset % 100)*60) return (yy, mm, dd, thh, tmm, tss, 0, 1, 0, tzoffset) def parsedate(data): """Convert a time string to a time tuple.""" t = parsedate_tz(data) if t is None: return t return t[:9] def mktime_tz(data): """Turn a 10-tuple as returned by parsedate_tz() into a UTC timestamp.""" if data[9] is None: # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: t = time.mktime(data[:8] + (0,)) return t - data[9] - time.timezone def formatdate(timeval=None): """Returns time format preferred for Internet standards. Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123 According to RFC 1123, day and month names must always be in English. If not for that, this code could use strftime(). It can't because strftime() honors the locale and could generate non-English names. """ if timeval is None: timeval = time.time() timeval = time.gmtime(timeval) return "%s, %02d %s %04d %02d:%02d:%02d GMT" % ( ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")[timeval[6]], timeval[2], ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")[timeval[1]-1], timeval[0], timeval[3], timeval[4], timeval[5]) # When used as script, run a small test program. # The first command line argument must be a filename containing one # message in RFC-822 format. if __name__ == '__main__': import sys, os file = os.path.join(os.environ['HOME'], 'Mail/inbox/1') if sys.argv[1:]: file = sys.argv[1] f = open(file, 'r') m = Message(f) print 'From:', m.getaddr('from') print 'To:', m.getaddrlist('to') print 'Subject:', m.getheader('subject') print 'Date:', m.getheader('date') date = m.getdate_tz('date') tz = date[-1] date = time.localtime(mktime_tz(date)) if date: print 'ParsedDate:', time.asctime(date), hhmmss = tz hhmm, ss = divmod(hhmmss, 60) hh, mm = divmod(hhmm, 60) print "%+03d%02d" % (hh, mm), if ss: print ".%02d" % ss, print else: print 'ParsedDate:', None m.rewindbody() n = 0 while f.readline(): n += 1 print 'Lines:', n print '-'*70 print 'len =', len(m) if 'Date' in m: print 'Date =', m['Date'] if 'X-Nonsense' in m: pass print 'keys =', m.keys() print 'values =', m.values() print 'items =', m.items() ================================================ FILE: third_party/stdlib/sched.py ================================================ """A generally useful event scheduler class. Each instance of this class manages its own queue. No multi-threading is implied; you are supposed to hack that yourself, or use a single instance per application. Each instance is parametrized with two functions, one that is supposed to return the current time, one that is supposed to implement a delay. You can implement real-time scheduling by substituting time and sleep from built-in module time, or you can implement simulated time by writing your own functions. This can also be used to integrate scheduling with STDWIN events; the delay function is allowed to modify the queue. Time can be expressed as integers or floating point numbers, as long as it is consistent. Events are specified by tuples (time, priority, action, argument). As in UNIX, lower priority numbers mean higher priority; in this way the queue can be maintained as a priority queue. Execution of the event means calling the action function, passing it the argument sequence in "argument" (remember that in Python, multiple function arguments are be packed in a sequence). The action function may be an instance method so it has another way to reference private data (besides global variables). """ # XXX The timefunc and delayfunc should have been defined as methods # XXX so you can define new kinds of schedulers using subclassing # XXX instead of having to define a module or class just to hold # XXX the global state of your particular time and delay functions. import heapq # TODO: grumpy modified version #from collections import namedtuple __all__ = ["scheduler"] # TODO: Use namedtuple # Event = namedtuple('Event', 'time, priority, action, argument') class Event(object): __slots__ = ['time', 'priority', 'action', 'argument'] def __init__(self, time, priority, action, argument): self.time = time self.priority = priority self.action = action self.argument = argument def get_fields(self): return (self.time, self.priority, self.action, self.argument) def __eq__(s, o): return (s.time, s.priority) == (o.time, o.priority) def __lt__(s, o): return (s.time, s.priority) < (o.time, o.priority) def __le__(s, o): return (s.time, s.priority) <= (o.time, o.priority) def __gt__(s, o): return (s.time, s.priority) > (o.time, o.priority) def __ge__(s, o): return (s.time, s.priority) >= (o.time, o.priority) class scheduler(object): def __init__(self, timefunc, delayfunc): """Initialize a new instance, passing the time and delay functions""" self._queue = [] self.timefunc = timefunc self.delayfunc = delayfunc def enterabs(self, time, priority, action, argument): """Enter a new event in the queue at an absolute time. Returns an ID for the event which can be used to remove it, if necessary. """ event = Event(time, priority, action, argument) heapq.heappush(self._queue, event) return event # The ID def enter(self, delay, priority, action, argument): """A variant that specifies the time as a relative time. This is actually the more commonly used interface. """ time = self.timefunc() + delay return self.enterabs(time, priority, action, argument) def cancel(self, event): """Remove an event from the queue. This must be presented the ID as returned by enter(). If the event is not in the queue, this raises ValueError. """ self._queue.remove(event) heapq.heapify(self._queue) def empty(self): """Check whether the queue is empty.""" return not self._queue def run(self): """Execute events until the queue is empty. When there is a positive delay until the first event, the delay function is called and the event is left in the queue; otherwise, the event is removed from the queue and executed (its action function is called, passing it the argument). If the delay function returns prematurely, it is simply restarted. It is legal for both the delay function and the action function to modify the queue or to raise an exception; exceptions are not caught but the scheduler's state remains well-defined so run() may be called again. A questionable hack is added to allow other threads to run: just after an event is executed, a delay of 0 is executed, to avoid monopolizing the CPU when other threads are also runnable. """ # localize variable access to minimize overhead # and to improve thread safety q = self._queue delayfunc = self.delayfunc timefunc = self.timefunc pop = heapq.heappop while q: # TODO: modified part of grumpy version. checked_event = q[0] time, priority, action, argument = checked_event.get_fields() now = timefunc() if now < time: delayfunc(time - now) else: event = pop(q) # Verify that the event was not removed or altered # by another thread after we last looked at q[0]. if event is checked_event: action(*argument) delayfunc(0) # Let other threads run else: heapq.heappush(q, event) @property def queue(self): """An ordered list of upcoming events. Events are named tuples with fields for: time, priority, action, arguments """ # Use heapq to sort the queue rather than using 'sorted(self._queue)'. # With heapq, two events scheduled at the same time will show in # the actual order they would be retrieved. events = self._queue[:] return map(heapq.heappop, [events]*len(events)) ================================================ FILE: third_party/stdlib/sha.py ================================================ # $Id$ # # Copyright (C) 2005 Gregory P. Smith (greg@krypto.org) # Licensed to PSF under a Contributor Agreement. # import warnings # warnings.warn("the sha module is deprecated; use the hashlib module instead", # DeprecationWarning, 2) import _sha sha = _sha.new new = _sha.new blocksize = 1 # legacy value (wrong in any useful sense) digest_size = 20 digestsize = 20 ================================================ FILE: third_party/stdlib/sre_compile.py ================================================ # -*- coding: utf-8 -*- # # Secret Labs' Regular Expression Engine # # convert template to internal format # # Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. # # See the sre.py file for information on usage and redistribution. # """Internal support module for sre""" import sys import _sre import sre_parse # TODO: Support from foo import * syntax. import sre_constants for name in sre_constants.__all__: globals()[name] = getattr(sre_constants, name) assert _sre.MAGIC == MAGIC, "SRE module mismatch" if _sre.CODESIZE == 2: MAXCODE = 65535 else: MAXCODE = 0xFFFFFFFFL _LITERAL_CODES = set([LITERAL, NOT_LITERAL]) _REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT]) _SUCCESS_CODES = set([SUCCESS, FAILURE]) _ASSERT_CODES = set([ASSERT, ASSERT_NOT]) # Sets of lowercase characters which have the same uppercase. _equivalences = ( # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I (0x69, 0x131), # iı # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S (0x73, 0x17f), # sſ # MICRO SIGN, GREEK SMALL LETTER MU (0xb5, 0x3bc), # µμ # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI (0x345, 0x3b9, 0x1fbe), # \u0345ιι # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL (0x3b2, 0x3d0), # βϐ # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL (0x3b5, 0x3f5), # εϵ # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL (0x3b8, 0x3d1), # θϑ # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL (0x3ba, 0x3f0), # κϰ # GREEK SMALL LETTER PI, GREEK PI SYMBOL (0x3c0, 0x3d6), # πϖ # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL (0x3c1, 0x3f1), # ρϱ # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA (0x3c2, 0x3c3), # ςσ # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL (0x3c6, 0x3d5), # φϕ # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE (0x1e61, 0x1e9b), # ṡẛ ) # Maps the lowercase code to lowercase codes which have the same uppercase. _ignorecase_fixes = {i: tuple(j for j in t if i != j) for t in _equivalences for i in t} def _compile(code, pattern, flags): # internal: compile a (sub)pattern emit = code.append _len = len LITERAL_CODES = _LITERAL_CODES REPEATING_CODES = _REPEATING_CODES SUCCESS_CODES = _SUCCESS_CODES ASSERT_CODES = _ASSERT_CODES if (flags & SRE_FLAG_IGNORECASE and not (flags & SRE_FLAG_LOCALE) and flags & SRE_FLAG_UNICODE): fixes = _ignorecase_fixes else: fixes = None for op, av in pattern: if op in LITERAL_CODES: if flags & SRE_FLAG_IGNORECASE: lo = _sre.getlower(av, flags) if fixes and lo in fixes: emit(OPCODES[IN_IGNORE]) skip = _len(code); emit(0) if op is NOT_LITERAL: emit(OPCODES[NEGATE]) for k in (lo,) + fixes[lo]: emit(OPCODES[LITERAL]) emit(k) emit(OPCODES[FAILURE]) code[skip] = _len(code) - skip else: emit(OPCODES[OP_IGNORE[op]]) emit(lo) else: emit(OPCODES[op]) emit(av) elif op is IN: if flags & SRE_FLAG_IGNORECASE: emit(OPCODES[OP_IGNORE[op]]) def fixup(literal, flags=flags): return _sre.getlower(literal, flags) else: emit(OPCODES[op]) fixup = None skip = _len(code); emit(0) _compile_charset(av, flags, code, fixup, fixes) code[skip] = _len(code) - skip elif op is ANY: if flags & SRE_FLAG_DOTALL: emit(OPCODES[ANY_ALL]) else: emit(OPCODES[ANY]) elif op in REPEATING_CODES: if flags & SRE_FLAG_TEMPLATE: raise error, "internal: unsupported template operator" emit(OPCODES[REPEAT]) skip = _len(code); emit(0) emit(av[0]) emit(av[1]) _compile(code, av[2], flags) emit(OPCODES[SUCCESS]) code[skip] = _len(code) - skip elif _simple(av) and op is not REPEAT: if op is MAX_REPEAT: emit(OPCODES[REPEAT_ONE]) else: emit(OPCODES[MIN_REPEAT_ONE]) skip = _len(code); emit(0) emit(av[0]) emit(av[1]) _compile(code, av[2], flags) emit(OPCODES[SUCCESS]) code[skip] = _len(code) - skip else: emit(OPCODES[REPEAT]) skip = _len(code); emit(0) emit(av[0]) emit(av[1]) _compile(code, av[2], flags) code[skip] = _len(code) - skip if op is MAX_REPEAT: emit(OPCODES[MAX_UNTIL]) else: emit(OPCODES[MIN_UNTIL]) elif op is SUBPATTERN: if av[0]: emit(OPCODES[MARK]) emit((av[0]-1)*2) # _compile_info(code, av[1], flags) _compile(code, av[1], flags) if av[0]: emit(OPCODES[MARK]) emit((av[0]-1)*2+1) elif op in SUCCESS_CODES: emit(OPCODES[op]) elif op in ASSERT_CODES: emit(OPCODES[op]) skip = _len(code); emit(0) if av[0] >= 0: emit(0) # look ahead else: lo, hi = av[1].getwidth() if lo != hi: raise error, "look-behind requires fixed-width pattern" emit(lo) # look behind _compile(code, av[1], flags) emit(OPCODES[SUCCESS]) code[skip] = _len(code) - skip elif op is CALL: emit(OPCODES[op]) skip = _len(code); emit(0) _compile(code, av, flags) emit(OPCODES[SUCCESS]) code[skip] = _len(code) - skip elif op is AT: emit(OPCODES[op]) if flags & SRE_FLAG_MULTILINE: av = AT_MULTILINE.get(av, av) if flags & SRE_FLAG_LOCALE: av = AT_LOCALE.get(av, av) elif flags & SRE_FLAG_UNICODE: av = AT_UNICODE.get(av, av) emit(ATCODES[av]) elif op is BRANCH: emit(OPCODES[op]) tail = [] tailappend = tail.append for av in av[1]: skip = _len(code); emit(0) # _compile_info(code, av, flags) _compile(code, av, flags) emit(OPCODES[JUMP]) tailappend(_len(code)); emit(0) code[skip] = _len(code) - skip emit(0) # end of branch for tail in tail: code[tail] = _len(code) - tail elif op is CATEGORY: emit(OPCODES[op]) if flags & SRE_FLAG_LOCALE: av = CH_LOCALE[av] elif flags & SRE_FLAG_UNICODE: av = CH_UNICODE[av] emit(CHCODES[av]) elif op is GROUPREF: if flags & SRE_FLAG_IGNORECASE: emit(OPCODES[OP_IGNORE[op]]) else: emit(OPCODES[op]) emit(av-1) elif op is GROUPREF_EXISTS: emit(OPCODES[op]) emit(av[0]-1) skipyes = _len(code); emit(0) _compile(code, av[1], flags) if av[2]: emit(OPCODES[JUMP]) skipno = _len(code); emit(0) code[skipyes] = _len(code) - skipyes + 1 _compile(code, av[2], flags) code[skipno] = _len(code) - skipno else: code[skipyes] = _len(code) - skipyes + 1 else: raise ValueError, ("unsupported operand type", op) def _compile_charset(charset, flags, code, fixup=None, fixes=None): # compile charset subprogram emit = code.append for op, av in _optimize_charset(charset, fixup, fixes, flags & SRE_FLAG_UNICODE): emit(OPCODES[op]) if op is NEGATE: pass elif op is LITERAL: emit(av) elif op is RANGE: emit(av[0]) emit(av[1]) elif op is CHARSET: # code.extend(av) code += (av) elif op is BIGCHARSET: # code.extend(av) code += (av) elif op is CATEGORY: if flags & SRE_FLAG_LOCALE: emit(CHCODES[CH_LOCALE[av]]) elif flags & SRE_FLAG_UNICODE: emit(CHCODES[CH_UNICODE[av]]) else: emit(CHCODES[av]) else: raise error, "internal: unsupported set operator" emit(OPCODES[FAILURE]) def _optimize_charset(charset, fixup, fixes, isunicode): # internal: optimize character set out = [] tail = [] # charmap = bytearray(256) charmap = [0] * 256 for op, av in charset: while True: try: if op is LITERAL: if fixup: i = fixup(av) charmap[i] = 1 if fixes and i in fixes: for k in fixes[i]: charmap[k] = 1 else: charmap[av] = 1 elif op is RANGE: r = range(av[0], av[1]+1) if fixup: r = map(fixup, r) if fixup and fixes: for i in r: charmap[i] = 1 if i in fixes: for k in fixes[i]: charmap[k] = 1 else: for i in r: charmap[i] = 1 elif op is NEGATE: out.append((op, av)) else: tail.append((op, av)) except IndexError: if len(charmap) == 256: # character set contains non-UCS1 character codes charmap += b'\0' * 0xff00 continue # character set contains non-BMP character codes if fixup and isunicode and op is RANGE: lo, hi = av ranges = [av] # There are only two ranges of cased astral characters: # 10400-1044F (Deseret) and 118A0-118DF (Warang Citi). _fixup_range(max(0x10000, lo), min(0x11fff, hi), ranges, fixup) for lo, hi in ranges: if lo == hi: tail.append((LITERAL, hi)) else: tail.append((RANGE, (lo, hi))) else: tail.append((op, av)) break # compress character map runs = [] q = 0 def char_find(l, s, start): i = start while i < len(l): if l[i] == s: return i i += 1 return -1 while True: # p = charmap.find(b'\1', q) p = char_find(charmap, 1, q) if p < 0: break if len(runs) >= 2: runs = None break # q = charmap.find(b'\0', p) q = char_find(charmap, 0, p) if q < 0: runs.append((p, len(charmap))) break runs.append((p, q)) if runs is not None: # use literal/range for p, q in runs: if q - p == 1: out.append((LITERAL, p)) else: out.append((RANGE, (p, q - 1))) out += tail # if the case was changed or new representation is more compact if fixup or len(out) < len(charset): return out # else original character set is good enough return charset # use bitmap if len(charmap) == 256: data = _mk_bitmap(charmap) out.append((CHARSET, data)) out += tail return out # To represent a big charset, first a bitmap of all characters in the # set is constructed. Then, this bitmap is sliced into chunks of 256 # characters, duplicate chunks are eliminated, and each chunk is # given a number. In the compiled expression, the charset is # represented by a 32-bit word sequence, consisting of one word for # the number of different chunks, a sequence of 256 bytes (64 words) # of chunk numbers indexed by their original chunk position, and a # sequence of 256-bit chunks (8 words each). # Compression is normally good: in a typical charset, large ranges of # Unicode will be either completely excluded (e.g. if only cyrillic # letters are to be matched), or completely included (e.g. if large # subranges of Kanji match). These ranges will be represented by # chunks of all one-bits or all zero-bits. # Matching can be also done efficiently: the more significant byte of # the Unicode character is an index into the chunk number, and the # less significant byte is a bit index in the chunk (just like the # CHARSET matching). # In UCS-4 mode, the BIGCHARSET opcode still supports only subsets # of the basic multilingual plane; an efficient representation # for all of Unicode has not yet been developed. # charmap = bytes(charmap) # should be hashable charmap = str(charmap) # should be hashable comps = {} # mapping = bytearray(256) mapping = [0] * 256 block = 0 # data = bytearray() data = [] for i in range(0, 65536, 256): chunk = charmap[i: i + 256] if chunk in comps: mapping[i // 256] = comps[chunk] else: mapping[i // 256] = comps[chunk] = block block += 1 data += chunk data = _mk_bitmap(data) data[0:0] = [block] + _bytes_to_codes(mapping) out.append((BIGCHARSET, data)) out += tail return out def _fixup_range(lo, hi, ranges, fixup): for i in map(fixup, range(lo, hi+1)): for k, (lo, hi) in enumerate(ranges): if i < lo: if l == lo - 1: ranges[k] = (i, hi) else: ranges.insert(k, (i, i)) break elif i > hi: if i == hi + 1: ranges[k] = (lo, i) break else: break else: ranges.append((i, i)) _CODEBITS = _sre.CODESIZE * 8 _BITS_TRANS = b'0' + b'1' * 255 # def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int): # s = bytes(bits).translate(_BITS_TRANS)[::-1] # r = [_int(s[i - _CODEBITS: i], 2) # for i in range(len(s), 0, -_CODEBITS)] # return r def _mk_bitmap(bits): data = [] dataappend = data.append # if _sre.CODESIZE == 2: # start = (1, 0) # else: # start = (1, 0) start = (1, 0) m, v = start for c in bits: if c: v = v + m m = m + m if m > MAXCODE: dataappend(v) m, v = start return data def _bytes_to_codes(b): return b[:] # Convert block indices to word array # import array # if _sre.CODESIZE == 2: # code = 'H' # else: # code = 'I' # a = array.array(code, bytes(b)) # assert a.itemsize == _sre.CODESIZE # assert len(a) * a.itemsize == len(b) # return a.tolist() def _simple(av): # check if av is a "simple" operator lo, hi = av[2].getwidth() return lo == hi == 1 and av[2][0][0] != SUBPATTERN def _compile_info(code, pattern, flags): # internal: compile an info block. in the current version, # this contains min/max pattern width, and an optional literal # prefix or a character map lo, hi = pattern.getwidth() if lo == 0: return # not worth it # look for a literal prefix prefix = [] prefixappend = prefix.append prefix_skip = 0 charset = [] # not used charsetappend = charset.append if not (flags & SRE_FLAG_IGNORECASE): # look for literal prefix for op, av in pattern.data: if op is LITERAL: if len(prefix) == prefix_skip: prefix_skip = prefix_skip + 1 prefixappend(av) elif op is SUBPATTERN and len(av[1]) == 1: op, av = av[1][0] if op is LITERAL: prefixappend(av) else: break else: break # if no prefix, look for charset prefix if not prefix and pattern.data: op, av = pattern.data[0] if op is SUBPATTERN and av[1]: op, av = av[1][0] if op is LITERAL: charsetappend((op, av)) elif op is BRANCH: c = [] cappend = c.append for p in av[1]: if not p: break op, av = p[0] if op is LITERAL: cappend((op, av)) else: break else: charset = c elif op is BRANCH: c = [] cappend = c.append for p in av[1]: if not p: break op, av = p[0] if op is LITERAL: cappend((op, av)) else: break else: charset = c elif op is IN: charset = av ## if prefix: ## print "*** PREFIX", prefix, prefix_skip ## if charset: ## print "*** CHARSET", charset # add an info block emit = code.append emit(OPCODES[INFO]) skip = len(code); emit(0) # literal flag mask = 0 if prefix: mask = SRE_INFO_PREFIX if len(prefix) == prefix_skip == len(pattern.data): mask = mask + SRE_INFO_LITERAL elif charset: mask = mask + SRE_INFO_CHARSET emit(mask) # pattern length if lo < MAXCODE: emit(lo) else: emit(MAXCODE) prefix = prefix[:MAXCODE] if hi < MAXCODE: emit(hi) else: emit(0) # add literal prefix if prefix: emit(len(prefix)) # length emit(prefix_skip) # skip # code.extend(prefix) code += (prefix) # generate overlap table table = [-1] + ([0]*len(prefix)) for i in xrange(len(prefix)): table[i+1] = table[i]+1 while table[i+1] > 0 and prefix[i] != prefix[table[i+1]-1]: table[i+1] = table[table[i+1]-1]+1 # code.extend(table[1:]) # don't store first entry code += (table[1:]) # don't store first entry elif charset: _compile_charset(charset, flags, code) code[skip] = len(code) - skip try: unicode except NameError: STRING_TYPES = (type(""),) else: STRING_TYPES = (type(""), type(unicode(""))) def isstring(obj): for tp in STRING_TYPES: if isinstance(obj, tp): return 1 return 0 def _code(p, flags): flags = p.pattern.flags | flags code = [] # compile info block _compile_info(code, p, flags) # compile the pattern _compile(code, p.data, flags) code.append(OPCODES[SUCCESS]) return code def compile(p, flags=0): # internal: convert pattern list to internal format if isstring(p): pattern = p p = sre_parse.parse(p, flags) else: pattern = None code = _code(p, flags) # print code # XXX: get rid of this limitation! if p.pattern.groups > 100: raise AssertionError( "sorry, but this version only supports 100 named groups" ) # map in either direction groupindex = p.pattern.groupdict indexgroup = [None] * p.pattern.groups for k, i in groupindex.items(): indexgroup[i] = k return _sre.compile( pattern, flags | p.pattern.flags, code, p.pattern.groups-1, groupindex, indexgroup ) ================================================ FILE: third_party/stdlib/sre_constants.py ================================================ # # Secret Labs' Regular Expression Engine # # various symbols used by the regular expression engine. # run this script to update the _sre include files! # # Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. # # See the sre.py file for information on usage and redistribution. # """Internal support module for sre""" __all__ = [ 'ANY', 'ANY_ALL', 'ASSERT', 'ASSERT_NOT', 'AT', 'ATCODES', 'AT_BEGINNING', 'AT_BEGINNING_LINE', 'AT_BEGINNING_STRING', 'AT_BOUNDARY', 'AT_END', 'AT_END_LINE', 'AT_END_STRING', 'AT_LOCALE', 'AT_LOC_BOUNDARY', 'AT_LOC_NON_BOUNDARY', 'AT_MULTILINE', 'AT_NON_BOUNDARY', 'AT_UNICODE', 'AT_UNI_BOUNDARY', 'AT_UNI_NON_BOUNDARY', 'BIGCHARSET', 'BRANCH', 'CALL', 'CATEGORY', 'CATEGORY_DIGIT', 'CATEGORY_LINEBREAK', 'CATEGORY_LOC_NOT_WORD', 'CATEGORY_LOC_WORD', 'CATEGORY_NOT_DIGIT', 'CATEGORY_NOT_LINEBREAK', 'CATEGORY_NOT_SPACE', 'CATEGORY_NOT_WORD', 'CATEGORY_SPACE', 'CATEGORY_UNI_DIGIT', 'CATEGORY_UNI_LINEBREAK', 'CATEGORY_UNI_NOT_DIGIT', 'CATEGORY_UNI_NOT_LINEBREAK', 'CATEGORY_UNI_NOT_SPACE', 'CATEGORY_UNI_NOT_WORD', 'CATEGORY_UNI_SPACE', 'CATEGORY_UNI_WORD', 'CATEGORY_WORD', 'CHARSET', 'CHCODES', 'CH_LOCALE', 'CH_UNICODE', 'FAILURE', 'GROUPREF', 'GROUPREF_EXISTS', 'GROUPREF_IGNORE', 'IN', 'INFO', 'IN_IGNORE', 'JUMP', 'LITERAL', 'LITERAL_IGNORE', 'MAGIC', 'MARK', 'MAXREPEAT', 'MAX_REPEAT', 'MAX_UNTIL', 'MIN_REPEAT', 'MIN_REPEAT_ONE', 'MIN_UNTIL', 'NEGATE', 'NOT_LITERAL', 'NOT_LITERAL_IGNORE', 'OPCODES', 'OP_IGNORE', 'RANGE', 'REPEAT', 'REPEAT_ONE', 'SRE_FLAG_DOTALL', 'SRE_FLAG_IGNORECASE', 'SRE_FLAG_LOCALE', 'SRE_FLAG_MULTILINE', 'SRE_FLAG_TEMPLATE', 'SRE_FLAG_UNICODE', 'SRE_FLAG_VERBOSE', 'SRE_INFO_CHARSET', 'SRE_INFO_LITERAL', 'SRE_INFO_PREFIX', 'SUBPATTERN', 'SUCCESS', 'SRE_FLAG_DEBUG', 'MAXCODE', 'error' ] # update when constants are added or removed MAGIC = 20031017 MAXCODE = 65535 # try: # from _sre import MAXREPEAT # except ImportError: # import _sre # MAXREPEAT = _sre.MAXREPEAT = 65535 MAXREPEAT = 65535 # SRE standard exception (access as sre.error) # should this really be here? class error(Exception): pass # operators FAILURE = "failure" SUCCESS = "success" ANY = "any" ANY_ALL = "any_all" ASSERT = "assert" ASSERT_NOT = "assert_not" AT = "at" BIGCHARSET = "bigcharset" BRANCH = "branch" CALL = "call" CATEGORY = "category" CHARSET = "charset" GROUPREF = "groupref" GROUPREF_IGNORE = "groupref_ignore" GROUPREF_EXISTS = "groupref_exists" IN = "in" IN_IGNORE = "in_ignore" INFO = "info" JUMP = "jump" LITERAL = "literal" LITERAL_IGNORE = "literal_ignore" MARK = "mark" MAX_REPEAT = "max_repeat" MAX_UNTIL = "max_until" MIN_REPEAT = "min_repeat" MIN_UNTIL = "min_until" NEGATE = "negate" NOT_LITERAL = "not_literal" NOT_LITERAL_IGNORE = "not_literal_ignore" RANGE = "range" REPEAT = "repeat" REPEAT_ONE = "repeat_one" SUBPATTERN = "subpattern" MIN_REPEAT_ONE = "min_repeat_one" # positions AT_BEGINNING = "at_beginning" AT_BEGINNING_LINE = "at_beginning_line" AT_BEGINNING_STRING = "at_beginning_string" AT_BOUNDARY = "at_boundary" AT_NON_BOUNDARY = "at_non_boundary" AT_END = "at_end" AT_END_LINE = "at_end_line" AT_END_STRING = "at_end_string" AT_LOC_BOUNDARY = "at_loc_boundary" AT_LOC_NON_BOUNDARY = "at_loc_non_boundary" AT_UNI_BOUNDARY = "at_uni_boundary" AT_UNI_NON_BOUNDARY = "at_uni_non_boundary" # categories CATEGORY_DIGIT = "category_digit" CATEGORY_NOT_DIGIT = "category_not_digit" CATEGORY_SPACE = "category_space" CATEGORY_NOT_SPACE = "category_not_space" CATEGORY_WORD = "category_word" CATEGORY_NOT_WORD = "category_not_word" CATEGORY_LINEBREAK = "category_linebreak" CATEGORY_NOT_LINEBREAK = "category_not_linebreak" CATEGORY_LOC_WORD = "category_loc_word" CATEGORY_LOC_NOT_WORD = "category_loc_not_word" CATEGORY_UNI_DIGIT = "category_uni_digit" CATEGORY_UNI_NOT_DIGIT = "category_uni_not_digit" CATEGORY_UNI_SPACE = "category_uni_space" CATEGORY_UNI_NOT_SPACE = "category_uni_not_space" CATEGORY_UNI_WORD = "category_uni_word" CATEGORY_UNI_NOT_WORD = "category_uni_not_word" CATEGORY_UNI_LINEBREAK = "category_uni_linebreak" CATEGORY_UNI_NOT_LINEBREAK = "category_uni_not_linebreak" OPCODES = [ # failure=0 success=1 (just because it looks better that way :-) FAILURE, SUCCESS, ANY, ANY_ALL, ASSERT, ASSERT_NOT, AT, BRANCH, CALL, CATEGORY, CHARSET, BIGCHARSET, GROUPREF, GROUPREF_EXISTS, GROUPREF_IGNORE, IN, IN_IGNORE, INFO, JUMP, LITERAL, LITERAL_IGNORE, MARK, MAX_UNTIL, MIN_UNTIL, NOT_LITERAL, NOT_LITERAL_IGNORE, NEGATE, RANGE, REPEAT, REPEAT_ONE, SUBPATTERN, MIN_REPEAT_ONE ] ATCODES = [ AT_BEGINNING, AT_BEGINNING_LINE, AT_BEGINNING_STRING, AT_BOUNDARY, AT_NON_BOUNDARY, AT_END, AT_END_LINE, AT_END_STRING, AT_LOC_BOUNDARY, AT_LOC_NON_BOUNDARY, AT_UNI_BOUNDARY, AT_UNI_NON_BOUNDARY ] CHCODES = [ CATEGORY_DIGIT, CATEGORY_NOT_DIGIT, CATEGORY_SPACE, CATEGORY_NOT_SPACE, CATEGORY_WORD, CATEGORY_NOT_WORD, CATEGORY_LINEBREAK, CATEGORY_NOT_LINEBREAK, CATEGORY_LOC_WORD, CATEGORY_LOC_NOT_WORD, CATEGORY_UNI_DIGIT, CATEGORY_UNI_NOT_DIGIT, CATEGORY_UNI_SPACE, CATEGORY_UNI_NOT_SPACE, CATEGORY_UNI_WORD, CATEGORY_UNI_NOT_WORD, CATEGORY_UNI_LINEBREAK, CATEGORY_UNI_NOT_LINEBREAK ] def makedict(list): d = {} i = 0 for item in list: d[item] = i i = i + 1 return d OPCODES = makedict(OPCODES) ATCODES = makedict(ATCODES) CHCODES = makedict(CHCODES) # replacement operations for "ignore case" mode OP_IGNORE = { GROUPREF: GROUPREF_IGNORE, IN: IN_IGNORE, LITERAL: LITERAL_IGNORE, NOT_LITERAL: NOT_LITERAL_IGNORE } AT_MULTILINE = { AT_BEGINNING: AT_BEGINNING_LINE, AT_END: AT_END_LINE } AT_LOCALE = { AT_BOUNDARY: AT_LOC_BOUNDARY, AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY } AT_UNICODE = { AT_BOUNDARY: AT_UNI_BOUNDARY, AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY } CH_LOCALE = { CATEGORY_DIGIT: CATEGORY_DIGIT, CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT, CATEGORY_SPACE: CATEGORY_SPACE, CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE, CATEGORY_WORD: CATEGORY_LOC_WORD, CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD, CATEGORY_LINEBREAK: CATEGORY_LINEBREAK, CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK } CH_UNICODE = { CATEGORY_DIGIT: CATEGORY_UNI_DIGIT, CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT, CATEGORY_SPACE: CATEGORY_UNI_SPACE, CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE, CATEGORY_WORD: CATEGORY_UNI_WORD, CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD, CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK, CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK } # flags SRE_FLAG_TEMPLATE = 1 # template mode (disable backtracking) SRE_FLAG_IGNORECASE = 2 # case insensitive SRE_FLAG_LOCALE = 4 # honour system locale SRE_FLAG_MULTILINE = 8 # treat target as multiline string SRE_FLAG_DOTALL = 16 # treat target as a single string SRE_FLAG_UNICODE = 32 # use unicode locale SRE_FLAG_VERBOSE = 64 # ignore whitespace and comments SRE_FLAG_DEBUG = 128 # debugging # flags for INFO primitive SRE_INFO_PREFIX = 1 # has prefix SRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix) SRE_INFO_CHARSET = 4 # pattern starts with character from given set # if __name__ == "__main__": # def dump(f, d, prefix): # items = d.items() # items.sort(key=lambda a: a[1]) # for k, v in items: # f.write("#define %s_%s %s\n" % (prefix, k.upper(), v)) # f = open("sre_constants.h", "w") # f.write("""\ # /* # * Secret Labs' Regular Expression Engine # * # * regular expression matching engine # * # * NOTE: This file is generated by sre_constants.py. If you need # * to change anything in here, edit sre_constants.py and run it. # * # * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. # * # * See the _sre.c file for information on usage and redistribution. # */ # """) # f.write("#define SRE_MAGIC %d\n" % MAGIC) # dump(f, OPCODES, "SRE_OP") # dump(f, ATCODES, "SRE") # dump(f, CHCODES, "SRE") # f.write("#define SRE_FLAG_TEMPLATE %d\n" % SRE_FLAG_TEMPLATE) # f.write("#define SRE_FLAG_IGNORECASE %d\n" % SRE_FLAG_IGNORECASE) # f.write("#define SRE_FLAG_LOCALE %d\n" % SRE_FLAG_LOCALE) # f.write("#define SRE_FLAG_MULTILINE %d\n" % SRE_FLAG_MULTILINE) # f.write("#define SRE_FLAG_DOTALL %d\n" % SRE_FLAG_DOTALL) # f.write("#define SRE_FLAG_UNICODE %d\n" % SRE_FLAG_UNICODE) # f.write("#define SRE_FLAG_VERBOSE %d\n" % SRE_FLAG_VERBOSE) # f.write("#define SRE_INFO_PREFIX %d\n" % SRE_INFO_PREFIX) # f.write("#define SRE_INFO_LITERAL %d\n" % SRE_INFO_LITERAL) # f.write("#define SRE_INFO_CHARSET %d\n" % SRE_INFO_CHARSET) # f.close() # print "done" ================================================ FILE: third_party/stdlib/sre_parse.py ================================================ # # Secret Labs' Regular Expression Engine # # convert re-style regular expression to sre pattern # # Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. # # See the sre.py file for information on usage and redistribution. # """Internal support module for sre""" # XXX: show string offset and offending character for all errors import sys # from sre_constants import * import sre_constants for name in sre_constants.__all__: globals()[name] = getattr(sre_constants, name) SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" DIGITS = set("0123456789") OCTDIGITS = set("01234567") HEXDIGITS = set("0123456789abcdefABCDEF") WHITESPACE = set(" \t\n\r\v\f") ESCAPES = { r"\a": (LITERAL, ord("\a")), r"\b": (LITERAL, ord("\b")), r"\f": (LITERAL, ord("\f")), r"\n": (LITERAL, ord("\n")), r"\r": (LITERAL, ord("\r")), r"\t": (LITERAL, ord("\t")), r"\v": (LITERAL, ord("\v")), r"\\": (LITERAL, ord("\\")) } CATEGORIES = { r"\A": (AT, AT_BEGINNING_STRING), # start of string r"\b": (AT, AT_BOUNDARY), r"\B": (AT, AT_NON_BOUNDARY), r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]), r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]), r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]), r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]), r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]), r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]), r"\Z": (AT, AT_END_STRING), # end of string } FLAGS = { # standard flags "i": SRE_FLAG_IGNORECASE, "L": SRE_FLAG_LOCALE, "m": SRE_FLAG_MULTILINE, "s": SRE_FLAG_DOTALL, "x": SRE_FLAG_VERBOSE, # extensions "t": SRE_FLAG_TEMPLATE, "u": SRE_FLAG_UNICODE, } class Pattern(object): # master pattern object. keeps track of global attributes def __init__(self): self.flags = 0 self.open = [] self.groups = 1 self.groupdict = {} self.lookbehind = 0 def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 if name is not None: ogid = self.groupdict.get(name, None) if ogid is not None: raise error, ("redefinition of group name %s as group %d; " "was group %d" % (repr(name), gid, ogid)) self.groupdict[name] = gid self.open.append(gid) return gid def closegroup(self, gid): # self.open.remove(gid) self.open = [x for x in self.open if x != gid] def checkgroup(self, gid): return gid < self.groups and gid not in self.open class SubPattern(object): # a subpattern, in intermediate form def __init__(self, pattern, data=None): self.pattern = pattern if data is None: data = [] self.data = data self.width = None def dump(self, level=0): seqtypes = (tuple, list) for op, av in self.data: print level*" " + op, if op == IN: # member sublanguage print for op, a in av: print (level+1)*" " + op, a elif op == BRANCH: print for i, a in enumerate(av[1]): if i: print level*" " + "or" a.dump(level+1) elif op == GROUPREF_EXISTS: condgroup, item_yes, item_no = av print condgroup item_yes.dump(level+1) if item_no: print level*" " + "else" item_no.dump(level+1) elif isinstance(av, seqtypes): nl = 0 for a in av: if isinstance(a, SubPattern): if not nl: print a.dump(level+1) nl = 1 else: print a, nl = 0 if not nl: print else: print av def __repr__(self): return repr(self.data) def __len__(self): return len(self.data) def __delitem__(self, index): # del self.data[index] self.data = self.data[:index] + self.data[index+1:] def __getitem__(self, index): if isinstance(index, slice): return SubPattern(self.pattern, self.data[index]) return self.data[index] def __setitem__(self, index, code): self.data[index] = code def insert(self, index, code): self.data.insert(index, code) def append(self, code): self.data.append(code) def getwidth(self): # determine the width (min, max) for this subpattern if self.width: return self.width lo = hi = 0 UNITCODES = (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY) REPEATCODES = (MIN_REPEAT, MAX_REPEAT) for op, av in self.data: if op is BRANCH: i = MAXREPEAT - 1 j = 0 for av in av[1]: l, h = av.getwidth() i = min(i, l) j = max(j, h) lo = lo + i hi = hi + j elif op is CALL: i, j = av.getwidth() lo = lo + i hi = hi + j elif op is SUBPATTERN: i, j = av[1].getwidth() lo = lo + i hi = hi + j elif op in REPEATCODES: i, j = av[2].getwidth() lo = lo + i * av[0] hi = hi + j * av[1] elif op in UNITCODES: lo = lo + 1 hi = hi + 1 elif op == SUCCESS: break self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT) return self.width class Tokenizer(object): def __init__(self, string): self.string = string self.index = 0 self.__next() def __next(self): if self.index >= len(self.string): self.next = None return char = self.string[self.index] if char[0] == "\\": try: c = self.string[self.index + 1] except IndexError: raise error, "bogus escape (end of line)" char = char + c self.index = self.index + len(char) self.next = char def match(self, char, skip=1): if char == self.next: if skip: self.__next() return 1 return 0 def get(self): this = self.next self.__next() return this def tell(self): return self.index, self.next def seek(self, index): self.index, self.next = index def isident(char): return "a" <= char <= "z" or "A" <= char <= "Z" or char == "_" def isdigit(char): return "0" <= char <= "9" def isname(name): # check that group name is a valid string if not isident(name[0]): return False for char in name[1:]: if not isident(char) and not isdigit(char): return False return True def _class_escape(source, escape): # handle escape code inside character class code = ESCAPES.get(escape) if code: return code code = CATEGORIES.get(escape) if code and code[0] == IN: return code try: c = escape[1:2] if c == "x": # hexadecimal escape (exactly two digits) while source.next in HEXDIGITS and len(escape) < 4: escape = escape + source.get() escape = escape[2:] if len(escape) != 2: raise error, "bogus escape: %s" % repr("\\" + escape) return LITERAL, int(escape, 16) & 0xff elif c in OCTDIGITS: # octal escape (up to three digits) while source.next in OCTDIGITS and len(escape) < 4: escape = escape + source.get() escape = escape[1:] return LITERAL, int(escape, 8) & 0xff elif c in DIGITS: raise error, "bogus escape: %s" % repr(escape) if len(escape) == 2: return LITERAL, ord(escape[1]) except ValueError: pass raise error, "bogus escape: %s" % repr(escape) def _escape(source, escape, state): # handle escape code in expression code = CATEGORIES.get(escape) if code: return code code = ESCAPES.get(escape) if code: return code try: c = escape[1:2] if c == "x": # hexadecimal escape while source.next in HEXDIGITS and len(escape) < 4: escape = escape + source.get() if len(escape) != 4: raise ValueError return LITERAL, int(escape[2:], 16) & 0xff elif c == "0": # octal escape while source.next in OCTDIGITS and len(escape) < 4: escape = escape + source.get() return LITERAL, int(escape[1:], 8) & 0xff elif c in DIGITS: # octal escape *or* decimal group reference (sigh) if source.next in DIGITS: escape = escape + source.get() if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and source.next in OCTDIGITS): # got three octal digits; this is an octal escape escape = escape + source.get() return LITERAL, int(escape[1:], 8) & 0xff # not an octal escape, so this is a group reference group = int(escape[1:]) if group < state.groups: if not state.checkgroup(group): raise error, "cannot refer to open group" # if state.lookbehind: # import warnings # warnings.warn('group references in lookbehind ' # 'assertions are not supported', # RuntimeWarning) return GROUPREF, group raise ValueError if len(escape) == 2: return LITERAL, ord(escape[1]) except ValueError: pass raise error, "bogus escape: %s" % repr(escape) def _parse_sub(source, state, nested=1): # parse an alternation: a|b|c items = [] itemsappend = items.append sourcematch = source.match while 1: itemsappend(_parse(source, state)) if sourcematch("|"): continue if not nested: break if not source.next or sourcematch(")", 0): break else: raise error, "pattern not properly closed" if len(items) == 1: return items[0] subpattern = SubPattern(state) subpatternappend = subpattern.append # check if all items share a common prefix while 1: prefix, common = None, False for item in items: if not item: break if prefix is None: prefix = item[0] elif item[0] != prefix: break else: # all subitems start with a common "prefix". # move it out of the branch # for item in items: # print "del", item[0], items # del item[0] for i in range(len(items)): items[i] = items[i][1:] subpatternappend(prefix) # continue # check next one common = True if common: continue break # check if the branch can be replaced by a character set for item in items: if len(item) != 1 or item[0][0] != LITERAL: break else: # we can store this as a character set instead of a # branch (the compiler may optimize this even more) set = [] setappend = set.append for item in items: setappend(item[0]) subpatternappend((IN, set)) return subpattern subpattern.append((BRANCH, (None, items))) return subpattern def _parse_sub_cond(source, state, condgroup): item_yes = _parse(source, state) if source.match("|"): item_no = _parse(source, state) if source.match("|"): raise error, "conditional backref with more than two branches" else: item_no = None if source.next and not source.match(")", 0): raise error, "pattern not properly closed" subpattern = SubPattern(state) subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) return subpattern _PATTERNENDERS = set("|)") _ASSERTCHARS = set("=!<") _LOOKBEHINDASSERTCHARS = set("=!") _REPEATCODES = set([MIN_REPEAT, MAX_REPEAT]) def _parse(source, state): # parse a simple pattern subpattern = SubPattern(state) # precompute constants into local variables subpatternappend = subpattern.append sourceget = source.get sourcematch = source.match _len = len PATTERNENDERS = _PATTERNENDERS ASSERTCHARS = _ASSERTCHARS LOOKBEHINDASSERTCHARS = _LOOKBEHINDASSERTCHARS REPEATCODES = _REPEATCODES while 1: if source.next in PATTERNENDERS: break # end of subpattern this = sourceget() if this is None: break # end of pattern if state.flags & SRE_FLAG_VERBOSE: # skip whitespace and comments if this in WHITESPACE: continue if this == "#": while 1: this = sourceget() if this in (None, "\n"): break continue if this and this[0] not in SPECIAL_CHARS: subpatternappend((LITERAL, ord(this))) elif this == "[": # character set set = [] setappend = set.append ## if sourcematch(":"): ## pass # handle character classes if sourcematch("^"): setappend((NEGATE, None)) # check remaining characters start = set[:] while 1: this = sourceget() if this == "]" and set != start: break elif this and this[0] == "\\": code1 = _class_escape(source, this) elif this: code1 = LITERAL, ord(this) else: raise error, "unexpected end of regular expression" if sourcematch("-"): # potential range this = sourceget() if this == "]": if code1[0] is IN: code1 = code1[1][0] setappend(code1) setappend((LITERAL, ord("-"))) break elif this: if this[0] == "\\": code2 = _class_escape(source, this) else: code2 = LITERAL, ord(this) if code1[0] != LITERAL or code2[0] != LITERAL: raise error, "bad character range" lo = code1[1] hi = code2[1] if hi < lo: raise error, "bad character range" setappend((RANGE, (lo, hi))) else: raise error, "unexpected end of regular expression" else: if code1[0] is IN: code1 = code1[1][0] setappend(code1) # XXX: should move set optimization to compiler! if _len(set)==1 and set[0][0] is LITERAL: subpatternappend(set[0]) # optimization elif _len(set)==2 and set[0][0] is NEGATE and set[1][0] is LITERAL: subpatternappend((NOT_LITERAL, set[1][1])) # optimization else: # XXX: should add charmap optimization here subpatternappend((IN, set)) elif this and this[0] in REPEAT_CHARS: # repeat previous item if this == "?": min, max = 0, 1 elif this == "*": min, max = 0, MAXREPEAT elif this == "+": min, max = 1, MAXREPEAT elif this == "{": if source.next == "}": subpatternappend((LITERAL, ord(this))) continue here = source.tell() min, max = 0, MAXREPEAT lo = hi = "" while source.next in DIGITS: lo = lo + source.get() if sourcematch(","): while source.next in DIGITS: hi = hi + sourceget() else: hi = lo if not sourcematch("}"): subpatternappend((LITERAL, ord(this))) source.seek(here) continue if lo: min = int(lo) if min >= MAXREPEAT: raise OverflowError("the repetition number is too large") if hi: max = int(hi) if max >= MAXREPEAT: raise OverflowError("the repetition number is too large") if max < min: raise error("bad repeat interval") else: raise error, "not supported" # figure out which item to repeat if subpattern: item = subpattern[-1:] else: item = None if not item or (_len(item) == 1 and item[0][0] == AT): raise error, "nothing to repeat" if item[0][0] in REPEATCODES: raise error, "multiple repeat" if sourcematch("?"): subpattern[-1] = (MIN_REPEAT, (min, max, item)) else: subpattern[-1] = (MAX_REPEAT, (min, max, item)) elif this == ".": subpatternappend((ANY, None)) elif this == "(": group = 1 name = None condgroup = None if sourcematch("?"): group = 0 # options if sourcematch("P"): # python extensions if sourcematch("<"): # named group: skip forward to end of name name = "" while 1: char = sourceget() if char is None: raise error, "unterminated name" if char == ">": break name = name + char group = 1 if not name: raise error("missing group name") if not isname(name): raise error("bad character in group name %r" % name) elif sourcematch("="): # named backreference name = "" while 1: char = sourceget() if char is None: raise error, "unterminated name" if char == ")": break name = name + char if not name: raise error("missing group name") if not isname(name): raise error("bad character in backref group name " "%r" % name) gid = state.groupdict.get(name) if gid is None: # msg = "unknown group name: {0!r}".format(name) msg = "unknown group name: %s" % (name) raise error(msg) # if state.lookbehind: # import warnings # warnings.warn('group references in lookbehind ' # 'assertions are not supported', # RuntimeWarning) subpatternappend((GROUPREF, gid)) continue else: char = sourceget() if char is None: raise error, "unexpected end of pattern" raise error, "unknown specifier: ?P%s" % char elif sourcematch(":"): # non-capturing group group = 2 elif sourcematch("#"): # comment while 1: if source.next is None or source.next == ")": break sourceget() if not sourcematch(")"): raise error, "unbalanced parenthesis" continue elif source.next in ASSERTCHARS: # lookahead assertions char = sourceget() dir = 1 if char == "<": if source.next not in LOOKBEHINDASSERTCHARS: raise error, "syntax error" dir = -1 # lookbehind char = sourceget() state.lookbehind += 1 p = _parse_sub(source, state) if dir < 0: state.lookbehind -= 1 if not sourcematch(")"): raise error, "unbalanced parenthesis" if char == "=": subpatternappend((ASSERT, (dir, p))) else: subpatternappend((ASSERT_NOT, (dir, p))) continue elif sourcematch("("): # conditional backreference group condname = "" while 1: char = sourceget() if char is None: raise error, "unterminated name" if char == ")": break condname = condname + char group = 2 if not condname: raise error("missing group name") if isname(condname): condgroup = state.groupdict.get(condname) if condgroup is None: # msg = "unknown group name: {0!r}".format(condname) msg = "unknown group name: %s" % (condname) raise error(msg) else: try: condgroup = int(condname) except ValueError: raise error, "bad character in group name" # if state.lookbehind: # import warnings # warnings.warn('group references in lookbehind ' # 'assertions are not supported', # RuntimeWarning) else: # flags if not source.next in FLAGS: raise error, "unexpected end of pattern" while source.next in FLAGS: state.flags = state.flags | FLAGS[sourceget()] if group: # parse group contents if group == 2: # anonymous group group = None else: group = state.opengroup(name) if condgroup: p = _parse_sub_cond(source, state, condgroup) else: p = _parse_sub(source, state) if not sourcematch(")"): raise error, "unbalanced parenthesis" if group is not None: state.closegroup(group) subpatternappend((SUBPATTERN, (group, p))) else: while 1: char = sourceget() if char is None: raise error, "unexpected end of pattern" if char == ")": break raise error, "unknown extension" elif this == "^": subpatternappend((AT, AT_BEGINNING)) elif this == "$": subpattern.append((AT, AT_END)) elif this and this[0] == "\\": code = _escape(source, this, state) subpatternappend(code) else: raise error, "parser error" return subpattern def parse(str, flags=0, pattern=None): # parse 're' pattern into list of (opcode, argument) tuples source = Tokenizer(str) if pattern is None: pattern = Pattern() pattern.flags = flags pattern.str = str p = _parse_sub(source, pattern, 0) tail = source.get() if tail == ")": raise error, "unbalanced parenthesis" elif tail: raise error, "bogus characters at end of regular expression" if not (flags & SRE_FLAG_VERBOSE) and p.pattern.flags & SRE_FLAG_VERBOSE: # the VERBOSE flag was switched on inside the pattern. to be # on the safe side, we'll parse the whole thing again... return parse(str, p.pattern.flags) if flags & SRE_FLAG_DEBUG: p.dump() return p def parse_template(source, pattern): # parse 're' replacement string into list of literals and # group references s = Tokenizer(source) sget = s.get p = [] a = p.append def literal(literal, p=p, pappend=a): if p and p[-1][0] is LITERAL: p[-1] = LITERAL, p[-1][1] + literal else: pappend((LITERAL, literal)) sep = source[:0] if type(sep) is type(""): makechar = chr else: makechar = unichr while 1: this = sget() if this is None: break # end of replacement string if this and this[0] == "\\": # group c = this[1:2] if c == "g": name = "" if s.match("<"): while 1: char = sget() if char is None: raise error, "unterminated group name" if char == ">": break name = name + char if not name: raise error, "missing group name" try: index = int(name) if index < 0: raise error, "negative group number" except ValueError: if not isname(name): raise error, "bad character in group name" try: index = pattern.groupindex[name] except KeyError: # msg = "unknown group name: {0!r}".format(name) msg = "unknown group name: %s" % (name) raise IndexError(msg) a((MARK, index)) elif c == "0": if s.next in OCTDIGITS: this = this + sget() if s.next in OCTDIGITS: this = this + sget() literal(makechar(int(this[1:], 8) & 0xff)) elif c in DIGITS: isoctal = False if s.next in DIGITS: this = this + sget() if (c in OCTDIGITS and this[2] in OCTDIGITS and s.next in OCTDIGITS): this = this + sget() isoctal = True literal(makechar(int(this[1:], 8) & 0xff)) if not isoctal: a((MARK, int(this[1:]))) else: try: this = makechar(ESCAPES[this][1]) except KeyError: pass literal(this) else: literal(this) # convert template to groups and literals lists i = 0 groups = [] groupsappend = groups.append literals = [None] * len(p) for c, s in p: if c is MARK: groupsappend((i, s)) # literal[i] is already None else: literals[i] = s i = i + 1 return groups, literals def expand_template(template, match): g = match.group sep = match.string[:0] groups, literals = template literals = literals[:] try: for index, group in groups: literals[index] = s = g(group) if s is None: raise error, "unmatched group" except IndexError: raise error, "invalid group reference" return sep.join(literals) ================================================ FILE: third_party/stdlib/string.py ================================================ """A collection of string operations (most are no longer used). Warning: most of the code you see here isn't normally used nowadays. Beginning with Python 1.6, many of these functions are implemented as methods on the standard string object. They used to be implemented by a built-in module called strop, but strop is now obsolete itself. Public module variables: whitespace -- a string containing all characters considered whitespace lowercase -- a string containing all characters considered lowercase letters uppercase -- a string containing all characters considered uppercase letters letters -- a string containing all characters considered letters digits -- a string containing all characters considered decimal digits hexdigits -- a string containing all characters considered hexadecimal digits octdigits -- a string containing all characters considered octal digits punctuation -- a string containing all characters considered punctuation printable -- a string containing all characters considered printable """ # Some strings for ctype-style character classification whitespace = ' \t\n\r\v\f' lowercase = 'abcdefghijklmnopqrstuvwxyz' uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' letters = lowercase + uppercase ascii_lowercase = lowercase ascii_uppercase = uppercase ascii_letters = ascii_lowercase + ascii_uppercase digits = '0123456789' hexdigits = digits + 'abcdef' + 'ABCDEF' octdigits = '01234567' punctuation = """!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" printable = digits + letters + punctuation + whitespace # Case conversion helpers # Use str to convert Unicode literal in case of -U # TODO: use map once implemented # l = map(chr, xrange(256)) l = [chr(x) for x in xrange(256)] _idmap = str('').join(l) del l # Functions which aren't available as string methods. # Capitalize the words in a string, e.g. " aBc dEf " -> "Abc Def". def capwords(s, sep=None): """capwords(s [,sep]) -> string Split the argument into words using split, capitalize each word using capitalize, and join the capitalized words using join. If the optional second argument sep is absent or None, runs of whitespace characters are replaced by a single space and leading and trailing whitespace are removed, otherwise sep is used to split and join the words. """ return (sep or ' ').join(x.capitalize() for x in s.split(sep)) # Construct a translation string _idmapL = None def maketrans(fromstr, tostr): """maketrans(frm, to) -> string Return a translation table (a string of 256 bytes long) suitable for use in string.translate. The strings frm and to must be of the same length. """ if len(fromstr) != len(tostr): raise ValueError, "maketrans arguments must have same length" global _idmapL if not _idmapL: _idmapL = list(_idmap) L = _idmapL[:] fromstr = map(ord, fromstr) for i in range(len(fromstr)): L[fromstr[i]] = tostr[i] return ''.join(L) #################################################################### import re as _re class _multimap(object): """Helper class for combining multiple mappings. Used by .{safe_,}substitute() to combine the mapping and keyword arguments. """ def __init__(self, primary, secondary): self._primary = primary self._secondary = secondary def __getitem__(self, key): try: return self._primary[key] except KeyError: return self._secondary[key] class _TemplateMetaclass(type): # pattern = r""" # %(delim)s(?: # (?P%(delim)s) | # Escape sequence of two delimiters # (?P%(id)s) | # delimiter and a Python identifier # {(?P%(id)s)} | # delimiter and a braced identifier # (?P) # Other ill-formed delimiter exprs # ) # """ pattern = r""" %s(?: (?P%s) | # Escape sequence of two delimiters (?P%s) | # delimiter and a Python identifier {(?P%s)} | # delimiter and a braced identifier (?P) # Other ill-formed delimiter exprs ) """ def __init__(cls, name, bases, dct): # super(_TemplateMetaclass, cls).__init__(name, bases, dct) super(_TemplateMetaclass, cls) if 'pattern' in dct: pattern = cls.pattern else: # pattern = _TemplateMetaclass.pattern % { # 'delim' : _re.escape(cls.delimiter), # 'id' : cls.idpattern, # } cls_delim, cls_id = _re.escape(cls.delimiter), cls.idpattern pattern = _TemplateMetaclass.pattern % (cls_delim, cls_delim, cls_id, cls_id) cls.pattern = _re.compile(pattern, _re.IGNORECASE | _re.VERBOSE) class Template(object): """A string class for supporting $-substitutions.""" __metaclass__ = _TemplateMetaclass delimiter = '$' idpattern = r'[_a-z][_a-z0-9]*' def __init__(self, template, *arg): self.template = template # Search for $$, $identifier, ${identifier}, and any bare $'s def _invalid(self, mo): i = mo.start('invalid') lines = self.template[:i].splitlines(True) if not lines: colno = 1 lineno = 1 else: colno = i - len(''.join(lines[:-1])) lineno = len(lines) raise ValueError('Invalid placeholder in string: line %d, col %d' % (lineno, colno)) def substitute(*args, **kws): if not args: raise TypeError("descriptor 'substitute' of 'Template' object " "needs an argument") self, args = args[0], args[1:] # allow the "self" keyword be passed if len(args) > 1: raise TypeError('Too many positional arguments') if not args: mapping = kws elif kws: mapping = _multimap(kws, args[0]) else: mapping = args[0] # Helper function for .sub() def convert(mo): # Check the most common path first. named = mo.group('named') or mo.group('braced') if named is not None: val = mapping[named] # We use this idiom instead of str() because the latter will # fail if val is a Unicode containing non-ASCII characters. return '%s' % (val,) if mo.group('escaped') is not None: return self.delimiter if mo.group('invalid') is not None: self._invalid(mo) raise ValueError('Unrecognized named group in pattern', self.pattern) return self.pattern.sub(convert, self.template) def safe_substitute(*args, **kws): if not args: raise TypeError("descriptor 'safe_substitute' of 'Template' object " "needs an argument") self, args = args[0], args[1:] # allow the "self" keyword be passed if len(args) > 1: raise TypeError('Too many positional arguments') if not args: mapping = kws elif kws: mapping = _multimap(kws, args[0]) else: mapping = args[0] # Helper function for .sub() def convert(mo): named = mo.group('named') or mo.group('braced') if named is not None: try: # We use this idiom instead of str() because the latter # will fail if val is a Unicode containing non-ASCII return '%s' % (mapping[named],) except KeyError: return mo.group() if mo.group('escaped') is not None: return self.delimiter if mo.group('invalid') is not None: return mo.group() raise ValueError('Unrecognized named group in pattern', self.pattern) return self.pattern.sub(convert, self.template) #################################################################### # NOTE: Everything below here is deprecated. Use string methods instead. # This stuff will go away in Python 3.0. # Backward compatible names for exceptions index_error = ValueError atoi_error = ValueError atof_error = ValueError atol_error = ValueError # convert UPPER CASE letters to lower case def lower(s): """lower(s) -> string Return a copy of the string s converted to lowercase. """ return s.lower() # Convert lower case letters to UPPER CASE def upper(s): """upper(s) -> string Return a copy of the string s converted to uppercase. """ return s.upper() # Swap lower case letters and UPPER CASE def swapcase(s): """swapcase(s) -> string Return a copy of the string s with upper case characters converted to lowercase and vice versa. """ return s.swapcase() # Strip leading and trailing tabs and spaces def strip(s, chars=None): """strip(s [,chars]) -> string Return a copy of the string s with leading and trailing whitespace removed. If chars is given and not None, remove characters in chars instead. If chars is unicode, S will be converted to unicode before stripping. """ return s.strip(chars) # Strip leading tabs and spaces def lstrip(s, chars=None): """lstrip(s [,chars]) -> string Return a copy of the string s with leading whitespace removed. If chars is given and not None, remove characters in chars instead. """ return s.lstrip(chars) # Strip trailing tabs and spaces def rstrip(s, chars=None): """rstrip(s [,chars]) -> string Return a copy of the string s with trailing whitespace removed. If chars is given and not None, remove characters in chars instead. """ return s.rstrip(chars) # Split a string into a list of space/tab-separated words def split(s, sep=None, maxsplit=-1): """split(s [,sep [,maxsplit]]) -> list of strings Return a list of the words in the string s, using sep as the delimiter string. If maxsplit is given, splits at no more than maxsplit places (resulting in at most maxsplit+1 words). If sep is not specified or is None, any whitespace string is a separator. (split and splitfields are synonymous) """ return s.split(sep, maxsplit) splitfields = split # Split a string into a list of space/tab-separated words def rsplit(s, sep=None, maxsplit=-1): """rsplit(s [,sep [,maxsplit]]) -> list of strings Return a list of the words in the string s, using sep as the delimiter string, starting at the end of the string and working to the front. If maxsplit is given, at most maxsplit splits are done. If sep is not specified or is None, any whitespace string is a separator. """ return s.rsplit(sep, maxsplit) # Join fields with optional separator def join(words, sep = ' '): """join(list [,sep]) -> string Return a string composed of the words in list, with intervening occurrences of sep. The default separator is a single space. (joinfields and join are synonymous) """ return sep.join(words) joinfields = join # Find substring, raise exception if not found def index(s, *args): """index(s, sub [,start [,end]]) -> int Like find but raises ValueError when the substring is not found. """ return s.index(*args) # Find last substring, raise exception if not found def rindex(s, *args): """rindex(s, sub [,start [,end]]) -> int Like rfind but raises ValueError when the substring is not found. """ return s.rindex(*args) # Count non-overlapping occurrences of substring def count(s, *args): """count(s, sub[, start[,end]]) -> int Return the number of occurrences of substring sub in string s[start:end]. Optional arguments start and end are interpreted as in slice notation. """ return s.count(*args) # Find substring, return -1 if not found def find(s, *args): """find(s, sub [,start [,end]]) -> in Return the lowest index in s where substring sub is found, such that sub is contained within s[start,end]. Optional arguments start and end are interpreted as in slice notation. Return -1 on failure. """ return s.find(*args) # Find last substring, return -1 if not found def rfind(s, *args): """rfind(s, sub [,start [,end]]) -> int Return the highest index in s where substring sub is found, such that sub is contained within s[start,end]. Optional arguments start and end are interpreted as in slice notation. Return -1 on failure. """ return s.rfind(*args) # for a bit of speed _float = float _int = int _long = long # Convert string to float def atof(s): """atof(s) -> float Return the floating point number represented by the string s. """ return _float(s) # Convert string to integer def atoi(s , base=10): """atoi(s [,base]) -> int Return the integer represented by the string s in the given base, which defaults to 10. The string s must consist of one or more digits, possibly preceded by a sign. If base is 0, it is chosen from the leading characters of s, 0 for octal, 0x or 0X for hexadecimal. If base is 16, a preceding 0x or 0X is accepted. """ return _int(s, base) # Convert string to long integer def atol(s, base=10): """atol(s [,base]) -> long Return the long integer represented by the string s in the given base, which defaults to 10. The string s must consist of one or more digits, possibly preceded by a sign. If base is 0, it is chosen from the leading characters of s, 0 for octal, 0x or 0X for hexadecimal. If base is 16, a preceding 0x or 0X is accepted. A trailing L or l is not accepted, unless base is 0. """ return _long(s, base) # Left-justify a string def ljust(s, width, *args): """ljust(s, width[, fillchar]) -> string Return a left-justified version of s, in a field of the specified width, padded with spaces as needed. The string is never truncated. If specified the fillchar is used instead of spaces. """ return s.ljust(width, *args) # Right-justify a string def rjust(s, width, *args): """rjust(s, width[, fillchar]) -> string Return a right-justified version of s, in a field of the specified width, padded with spaces as needed. The string is never truncated. If specified the fillchar is used instead of spaces. """ return s.rjust(width, *args) # Center a string def center(s, width, *args): """center(s, width[, fillchar]) -> string Return a center version of s, in a field of the specified width. padded with spaces as needed. The string is never truncated. If specified the fillchar is used instead of spaces. """ return s.center(width, *args) # Zero-fill a number, e.g., (12, 3) --> '012' and (-3, 3) --> '-03' # Decadent feature: the argument may be a string or a number # (Use of this is deprecated; it should be a string as with ljust c.s.) def zfill(x, width): """zfill(x, width) -> string Pad a numeric string x with zeros on the left, to fill a field of the specified width. The string x is never truncated. """ if not isinstance(x, basestring): x = repr(x) return x.zfill(width) # Expand tabs in a string. # Doesn't take non-printing chars into account, but does understand \n. def expandtabs(s, tabsize=8): """expandtabs(s [,tabsize]) -> string Return a copy of the string s with all tab characters replaced by the appropriate number of spaces, depending on the current column, and the tabsize (default 8). """ return s.expandtabs(tabsize) # Character translation through look-up table. def translate(s, table, deletions=""): """translate(s,table [,deletions]) -> string Return a copy of the string s, where all characters occurring in the optional argument deletions are removed, and the remaining characters have been mapped through the given translation table, which must be a string of length 256. The deletions argument is not allowed for Unicode strings. """ if deletions or table is None: return s.translate(table, deletions) else: # Add s[:0] so that if s is Unicode and table is an 8-bit string, # table is converted to Unicode. This means that table *cannot* # be a dictionary -- for that feature, use u.translate() directly. return s.translate(table + s[:0]) # Capitalize a string, e.g. "aBc dEf" -> "Abc def". def capitalize(s): """capitalize(s) -> string Return a copy of the string s with only its first character capitalized. """ return s.capitalize() # Substring replacement (global) def replace(s, old, new, maxreplace=-1): """replace (str, old, new[, maxreplace]) -> string Return a copy of string str with all occurrences of substring old replaced by new. If the optional argument maxreplace is given, only the first maxreplace occurrences are replaced. """ return s.replace(old, new, maxreplace) # Try importing optional built-in module "strop" -- if it exists, # it redefines some string operations that are 100-1000 times faster. # It also defines values for whitespace, lowercase and uppercase # that match 's definitions. # try: # from strop import maketrans, lowercase, uppercase, whitespace # letters = lowercase + uppercase # except ImportError: # pass # Use the original versions ######################################################################## # the Formatter class # see PEP 3101 for details and purpose of this class # The hard parts are reused from the C implementation. They're exposed as "_" # prefixed methods of str and unicode. # The overall parser is implemented in str._formatter_parser. # The field name parser is implemented in str._formatter_field_name_split class Formatter(object): def format(*args, **kwargs): if not args: raise TypeError("descriptor 'format' of 'Formatter' object " "needs an argument") self, args = args[0], args[1:] # allow the "self" keyword be passed try: format_string, args = args[0], args[1:] # allow the "format_string" keyword be passed except IndexError: if 'format_string' in kwargs: format_string = kwargs.pop('format_string') else: raise TypeError("format() missing 1 required positional " "argument: 'format_string'") return self.vformat(format_string, args, kwargs) def vformat(self, format_string, args, kwargs): used_args = set() result = self._vformat(format_string, args, kwargs, used_args, 2) self.check_unused_args(used_args, args, kwargs) return result def _vformat(self, format_string, args, kwargs, used_args, recursion_depth): if recursion_depth < 0: raise ValueError('Max string recursion exceeded') result = [] for literal_text, field_name, format_spec, conversion in \ self.parse(format_string): # output the literal text if literal_text: result.append(literal_text) # if there's a field, output it if field_name is not None: # this is some markup, find the object and do # the formatting # given the field_name, find the object it references # and the argument it came from obj, arg_used = self.get_field(field_name, args, kwargs) used_args.add(arg_used) # do any conversion on the resulting object obj = self.convert_field(obj, conversion) # expand the format spec, if needed format_spec = self._vformat(format_spec, args, kwargs, used_args, recursion_depth-1) # format the object and append to the result result.append(self.format_field(obj, format_spec)) return ''.join(result) def get_value(self, key, args, kwargs): if isinstance(key, (int, long)): return args[key] else: return kwargs[key] def check_unused_args(self, used_args, args, kwargs): pass def format_field(self, value, format_spec): return format(value, format_spec) def convert_field(self, value, conversion): # do any conversion on the resulting object if conversion is None: return value elif conversion == 's': return str(value) elif conversion == 'r': return repr(value) raise ValueError("Unknown conversion specifier %s" % (conversion)) # returns an iterable that contains tuples of the form: # (literal_text, field_name, format_spec, conversion) # literal_text can be zero length # field_name can be None, in which case there's no # object to format and output # if field_name is not None, it is looked up, formatted # with format_spec and conversion and then used def parse(self, format_string): return format_string._formatter_parser() # given a field_name, find the object it references. # field_name: the field being looked up, e.g. "0.name" # or "lookup[3]" # used_args: a set of which args have been used # args, kwargs: as passed in to vformat def get_field(self, field_name, args, kwargs): first, rest = field_name._formatter_field_name_split() obj = self.get_value(first, args, kwargs) # loop through the rest of the field_name, doing # getattr or getitem as needed for is_attr, i in rest: if is_attr: obj = getattr(obj, i) else: obj = obj[i] return obj, first ================================================ FILE: third_party/stdlib/test/__init__.py ================================================ # Dummy file to make this directory a package. ================================================ FILE: third_party/stdlib/test/list_tests.py ================================================ """ Tests common to list and UserList.UserList """ import sys import os from test import test_support, seq_tests class CommonTest(seq_tests.CommonTest): # def test_init(self): # # Iterable arg is optional # self.assertEqual(self.type2test([]), self.type2test()) # # # Init clears previous values # a = self.type2test([1, 2, 3]) # a.__init__() # self.assertEqual(a, self.type2test([])) # # # Init overwrites previous values # a = self.type2test([1, 2, 3]) # a.__init__([4, 5, 6]) # self.assertEqual(a, self.type2test([4, 5, 6])) # # # Mutables always return a new object # b = self.type2test(a) # self.assertNotEqual(id(a), id(b)) # self.assertEqual(a, b) # def test_repr(self): # l0 = [] # l2 = [0, 1, 2] # a0 = self.type2test(l0) # a2 = self.type2test(l2) # # self.assertEqual(str(a0), str(l0)) # self.assertEqual(repr(a0), repr(l0)) # self.assertEqual(repr(a2), repr(l2)) # self.assertEqual(str(a2), "[0, 1, 2]") # self.assertEqual(repr(a2), "[0, 1, 2]") # # a2.append(a2) # a2.append(3) # self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") # self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") # # l0 = [] # for i in xrange(sys.getrecursionlimit() + 100): # l0 = [l0] # self.assertRaises(RuntimeError, repr, l0) # def test_print(self): # d = self.type2test(xrange(200)) # d.append(d) # d.extend(xrange(200,400)) # d.append(d) # d.append(400) # try: # with open(test_support.TESTFN, "wb") as fo: # print >> fo, d, # with open(test_support.TESTFN, "rb") as fo: # self.assertEqual(fo.read(), repr(d)) # finally: # os.remove(test_support.TESTFN) def test_set_subscript(self): a = self.type2test(range(20)) self.assertRaises(ValueError, a.__setitem__, slice(0, 10, 0), [1,2,3]) self.assertRaises(TypeError, a.__setitem__, slice(0, 10), 1) self.assertRaises(ValueError, a.__setitem__, slice(0, 10, 2), [1,2]) self.assertRaises(TypeError, a.__getitem__, 'x', 1) a[slice(2,10,3)] = [1,2,3] self.assertEqual(a, self.type2test([0, 1, 1, 3, 4, 2, 6, 7, 3, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19])) # def test_reversed(self): # a = self.type2test(range(20)) # r = reversed(a) # self.assertEqual(list(r), self.type2test(range(19, -1, -1))) # self.assertRaises(StopIteration, r.next) # self.assertEqual(list(reversed(self.type2test())), # self.type2test()) # # Bug 3689: make sure list-reversed-iterator doesn't have __len__ # self.assertRaises(TypeError, len, reversed([1,2,3])) def test_setitem(self): a = self.type2test([0, 1]) a[0] = 0 a[1] = 100 self.assertEqual(a, self.type2test([0, 100])) a[-1] = 200 self.assertEqual(a, self.type2test([0, 200])) a[-2] = 100 self.assertEqual(a, self.type2test([100, 200])) self.assertRaises(IndexError, a.__setitem__, -3, 200) self.assertRaises(IndexError, a.__setitem__, 2, 200) a = self.type2test([]) self.assertRaises(IndexError, a.__setitem__, 0, 200) self.assertRaises(IndexError, a.__setitem__, -1, 200) self.assertRaises(TypeError, a.__setitem__) a = self.type2test([0,1,2,3,4]) a[0L] = 1 a[1L] = 2 a[2L] = 3 self.assertEqual(a, self.type2test([1,2,3,3,4])) a[0] = 5 a[1] = 6 a[2] = 7 self.assertEqual(a, self.type2test([5,6,7,3,4])) a[-2L] = 88 a[-1L] = 99 self.assertEqual(a, self.type2test([5,6,7,88,99])) a[-2] = 8 a[-1] = 9 self.assertEqual(a, self.type2test([5,6,7,8,9])) def test_delitem(self): a = self.type2test([0, 1]) del a[1] self.assertEqual(a, [0]) del a[0] self.assertEqual(a, []) a = self.type2test([0, 1]) del a[-2] self.assertEqual(a, [1]) del a[-1] self.assertEqual(a, []) a = self.type2test([0, 1]) self.assertRaises(IndexError, a.__delitem__, -3) self.assertRaises(IndexError, a.__delitem__, 2) a = self.type2test([]) self.assertRaises(IndexError, a.__delitem__, 0) self.assertRaises(TypeError, a.__delitem__) # def test_setslice(self): # l = [0, 1] # a = self.type2test(l) # # for i in range(-3, 4): # a[:i] = l[:i] # self.assertEqual(a, l) # a2 = a[:] # a2[:i] = a[:i] # self.assertEqual(a2, a) # a[i:] = l[i:] # self.assertEqual(a, l) # a2 = a[:] # a2[i:] = a[i:] # self.assertEqual(a2, a) # for j in range(-3, 4): # a[i:j] = l[i:j] # self.assertEqual(a, l) # a2 = a[:] # a2[i:j] = a[i:j] # self.assertEqual(a2, a) # # aa2 = a2[:] # aa2[:0] = [-2, -1] # self.assertEqual(aa2, [-2, -1, 0, 1]) # aa2[0:] = [] # self.assertEqual(aa2, []) # # a = self.type2test([1, 2, 3, 4, 5]) # a[:-1] = a # self.assertEqual(a, self.type2test([1, 2, 3, 4, 5, 5])) # a = self.type2test([1, 2, 3, 4, 5]) # a[1:] = a # self.assertEqual(a, self.type2test([1, 1, 2, 3, 4, 5])) # a = self.type2test([1, 2, 3, 4, 5]) # a[1:-1] = a # self.assertEqual(a, self.type2test([1, 1, 2, 3, 4, 5, 5])) # # a = self.type2test([]) # a[:] = tuple(range(10)) # self.assertEqual(a, self.type2test(range(10))) # # self.assertRaises(TypeError, a.__setslice__, 0, 1, 5) # self.assertRaises(TypeError, a.__setitem__, slice(0, 1, 5)) # # self.assertRaises(TypeError, a.__setslice__) # self.assertRaises(TypeError, a.__setitem__) def test_delslice(self): a = self.type2test([0, 1]) del a[1:2] del a[0:1] self.assertEqual(a, self.type2test([])) a = self.type2test([0, 1]) del a[1L:2L] del a[0L:1L] self.assertEqual(a, self.type2test([])) a = self.type2test([0, 1]) del a[-2:-1] self.assertEqual(a, self.type2test([1])) a = self.type2test([0, 1]) del a[-2L:-1L] self.assertEqual(a, self.type2test([1])) a = self.type2test([0, 1]) del a[1:] del a[:1] self.assertEqual(a, self.type2test([])) a = self.type2test([0, 1]) del a[1L:] del a[:1L] self.assertEqual(a, self.type2test([])) a = self.type2test([0, 1]) del a[-1:] self.assertEqual(a, self.type2test([0])) a = self.type2test([0, 1]) del a[-1L:] self.assertEqual(a, self.type2test([0])) a = self.type2test([0, 1]) del a[:] self.assertEqual(a, self.type2test([])) def test_append(self): a = self.type2test([]) a.append(0) a.append(1) a.append(2) self.assertEqual(a, self.type2test([0, 1, 2])) self.assertRaises(TypeError, a.append) # def test_extend(self): # a1 = self.type2test([0]) # a2 = self.type2test((0, 1)) # a = a1[:] # a.extend(a2) # self.assertEqual(a, a1 + a2) # # a.extend(self.type2test([])) # self.assertEqual(a, a1 + a2) # # a.extend(a) # self.assertEqual(a, self.type2test([0, 0, 1, 0, 0, 1])) # # a = self.type2test("spam") # a.extend("eggs") # self.assertEqual(a, list("spameggs")) # # self.assertRaises(TypeError, a.extend, None) # # self.assertRaises(TypeError, a.extend) def test_insert(self): a = self.type2test([0, 1, 2]) a.insert(0, -2) a.insert(1, -1) a.insert(2, 0) self.assertEqual(a, [-2, -1, 0, 0, 1, 2]) b = a[:] b.insert(-2, "foo") b.insert(-200, "left") b.insert(200, "right") self.assertEqual(b, self.type2test(["left",-2,-1,0,0,"foo",1,2,"right"])) self.assertRaises(TypeError, a.insert) def test_pop(self): a = self.type2test([-1, 0, 1]) a.pop() self.assertEqual(a, [-1, 0]) a.pop(0) self.assertEqual(a, [0]) self.assertRaises(IndexError, a.pop, 5) a.pop(0) self.assertEqual(a, []) self.assertRaises(IndexError, a.pop) self.assertRaises(TypeError, a.pop, 42, 42) a = self.type2test([0, 10, 20, 30, 40]) def test_remove(self): a = self.type2test([0, 0, 1]) a.remove(1) self.assertEqual(a, [0, 0]) a.remove(0) self.assertEqual(a, [0]) a.remove(0) self.assertEqual(a, []) self.assertRaises(ValueError, a.remove, 0) self.assertRaises(TypeError, a.remove) class BadExc(Exception): pass class BadCmp(object): def __eq__(self, other): if other == 2: raise BadExc() return False a = self.type2test([0, 1, 2, 3]) self.assertRaises(BadExc, a.remove, BadCmp()) class BadCmp2(object): def __eq__(self, other): raise BadExc() d = self.type2test('abcdefghcij') d.remove('c') self.assertEqual(d, self.type2test('abdefghcij')) d.remove('c') self.assertEqual(d, self.type2test('abdefghij')) self.assertRaises(ValueError, d.remove, 'c') self.assertEqual(d, self.type2test('abdefghij')) # Handle comparison errors d = self.type2test(['a', 'b', BadCmp2(), 'c']) e = self.type2test(d) self.assertRaises(BadExc, d.remove, 'c') for x, y in zip(d, e): # verify that original order and values are retained. self.assertIs(x, y) def test_count(self): a = self.type2test([0, 1, 2])*3 self.assertEqual(a.count(0), 3) self.assertEqual(a.count(1), 3) self.assertEqual(a.count(3), 0) self.assertRaises(TypeError, a.count) class BadExc(Exception): pass class BadCmp(object): def __eq__(self, other): if other == 2: raise BadExc() return False self.assertRaises(BadExc, a.count, BadCmp()) # def test_index(self): # u = self.type2test([0, 1]) # self.assertEqual(u.index(0), 0) # self.assertEqual(u.index(1), 1) # self.assertRaises(ValueError, u.index, 2) # # u = self.type2test([-2, -1, 0, 0, 1, 2]) # self.assertEqual(u.count(0), 2) # self.assertEqual(u.index(0), 2) # self.assertEqual(u.index(0, 2), 2) # self.assertEqual(u.index(-2, -10), 0) # self.assertEqual(u.index(0, 3), 3) # self.assertEqual(u.index(0, 3, 4), 3) # self.assertRaises(ValueError, u.index, 2, 0, -10) # # self.assertRaises(TypeError, u.index) # # class BadExc(Exception): # pass # # class BadCmp(object): # def __eq__(self, other): # if other == 2: # raise BadExc() # return False # # a = self.type2test([0, 1, 2, 3]) # self.assertRaises(BadExc, a.index, BadCmp()) # # a = self.type2test([-2, -1, 0, 0, 1, 2]) # self.assertEqual(a.index(0), 2) # self.assertEqual(a.index(0, 2), 2) # self.assertEqual(a.index(0, -4), 2) # self.assertEqual(a.index(-2, -10), 0) # self.assertEqual(a.index(0, 3), 3) # self.assertEqual(a.index(0, -3), 3) # self.assertEqual(a.index(0, 3, 4), 3) # self.assertEqual(a.index(0, -3, -2), 3) # self.assertEqual(a.index(0, -4*sys.maxint, 4*sys.maxint), 2) # self.assertRaises(ValueError, a.index, 0, 4*sys.maxint,-4*sys.maxint) # self.assertRaises(ValueError, a.index, 2, 0, -10) # a.remove(0) # self.assertRaises(ValueError, a.index, 2, 0, 4) # self.assertEqual(a, self.type2test([-2, -1, 0, 1, 2])) # # # Test modifying the list during index's iteration # class EvilCmp(object): # def __init__(self, victim): # self.victim = victim # def __eq__(self, other): # del self.victim[:] # return False # a = self.type2test() # a[:] = [EvilCmp(a) for _ in xrange(100)] # # This used to seg fault before patch #1005778 # self.assertRaises(ValueError, a.index, None) def test_reverse(self): u = self.type2test([-2, -1, 0, 1, 2]) u2 = u[:] u.reverse() self.assertEqual(u, [2, 1, 0, -1, -2]) u.reverse() self.assertEqual(u, u2) self.assertRaises(TypeError, u.reverse, 42) # def test_sort(self): # with test_support.check_py3k_warnings( # ("the cmp argument is not supported", DeprecationWarning)): # self._test_sort() def _test_sort(self): u = self.type2test([1, 0]) u.sort() self.assertEqual(u, [0, 1]) u = self.type2test([2,1,0,-1,-2]) u.sort() self.assertEqual(u, self.type2test([-2,-1,0,1,2])) self.assertRaises(TypeError, u.sort, 42, 42) def revcmp(a, b): return cmp(b, a) u.sort(revcmp) self.assertEqual(u, self.type2test([2,1,0,-1,-2])) # The following dumps core in unpatched Python 1.5: def myComparison(x,y): return cmp(x%3, y%7) z = self.type2test(range(12)) z.sort(myComparison) self.assertRaises(TypeError, z.sort, 2) def selfmodifyingComparison(x,y): z.append(1) return cmp(x, y) self.assertRaises(ValueError, z.sort, selfmodifyingComparison) self.assertRaises(TypeError, z.sort, lambda x, y: 's') self.assertRaises(TypeError, z.sort, 42, 42, 42, 42) def test_slice(self): u = self.type2test("spam") u[:2] = "h" self.assertEqual(u, list("ham")) def test_iadd(self): super(CommonTest, self).test_iadd() u = self.type2test([0, 1]) u2 = u u += [2, 3] self.assertIs(u, u2) u = self.type2test("spam") u += "eggs" self.assertEqual(u, self.type2test("spameggs")) self.assertRaises(TypeError, u.__iadd__, None) def test_imul(self): u = self.type2test([0, 1]) u *= 3 self.assertEqual(u, self.type2test([0, 1, 0, 1, 0, 1])) u *= 0 self.assertEqual(u, self.type2test([])) s = self.type2test([]) oldid = id(s) s *= 10 self.assertEqual(id(s), oldid) # def test_extendedslicing(self): # # subscript # a = self.type2test([0,1,2,3,4]) # # # deletion # del a[::2] # self.assertEqual(a, self.type2test([1,3])) # a = self.type2test(range(5)) # del a[1::2] # self.assertEqual(a, self.type2test([0,2,4])) # a = self.type2test(range(5)) # del a[1::-2] # self.assertEqual(a, self.type2test([0,2,3,4])) # a = self.type2test(range(10)) # del a[::1000] # self.assertEqual(a, self.type2test([1, 2, 3, 4, 5, 6, 7, 8, 9])) # # assignment # a = self.type2test(range(10)) # a[::2] = [-1]*5 # self.assertEqual(a, self.type2test([-1, 1, -1, 3, -1, 5, -1, 7, -1, 9])) # a = self.type2test(range(10)) # a[::-4] = [10]*3 # self.assertEqual(a, self.type2test([0, 10, 2, 3, 4, 10, 6, 7, 8 ,10])) # a = self.type2test(range(4)) # a[::-1] = a # self.assertEqual(a, self.type2test([3, 2, 1, 0])) # a = self.type2test(range(10)) # b = a[:] # c = a[:] # a[2:3] = self.type2test(["two", "elements"]) # b[slice(2,3)] = self.type2test(["two", "elements"]) # c[2:3:] = self.type2test(["two", "elements"]) # self.assertEqual(a, b) # self.assertEqual(a, c) # a = self.type2test(range(10)) # a[::2] = tuple(range(5)) # self.assertEqual(a, self.type2test([0, 1, 1, 3, 2, 5, 3, 7, 4, 9])) # # test issue7788 # a = self.type2test(range(10)) # del a[9::1<<333] def test_constructor_exception_handling(self): # Bug #1242657 class F(object): def __iter__(self): raise KeyboardInterrupt self.assertRaises(KeyboardInterrupt, list, F()) def test_exhausted_iterator(self): a = self.type2test([1, 2, 3]) exhit = iter(a) empit = iter(a) for x in exhit: # exhaust the iterator next(empit) # not exhausted a.append(9) self.assertEqual(list(exhit), []) self.assertEqual(list(empit), [9]) self.assertEqual(a, self.type2test([1, 2, 3, 9])) ================================================ FILE: third_party/stdlib/test/lock_tests.py ================================================ """ Various tests for synchronization primitives. """ import sys import time from thread import start_new_thread, get_ident import threading import unittest from test import test_support as support def _wait(): # A crude wait/yield function not relying on synchronization primitives. time.sleep(0.01) class Bunch(object): """ A bunch of threads. """ def __init__(self, f, n, wait_before_exit=False): """ Construct a bunch of `n` threads running the same function `f`. If `wait_before_exit` is True, the threads won't terminate until do_finish() is called. """ self.f = f self.n = n self.started = [] self.finished = [] self._can_exit = not wait_before_exit def task(): tid = get_ident() self.started.append(tid) try: f() finally: self.finished.append(tid) while not self._can_exit: _wait() try: for i in range(n): start_new_thread(task, ()) except: self._can_exit = True raise def wait_for_started(self): while len(self.started) < self.n: _wait() def wait_for_finished(self): while len(self.finished) < self.n: _wait() def do_finish(self): self._can_exit = True class BaseTestCase(unittest.TestCase): def setUp(self): self._threads = support.threading_setup() def tearDown(self): support.threading_cleanup(*self._threads) support.reap_children() class BaseLockTests(BaseTestCase): """ Tests for both recursive and non-recursive locks. """ def test_constructor(self): lock = self.locktype() del lock def test_acquire_destroy(self): lock = self.locktype() lock.acquire() del lock def test_acquire_release(self): lock = self.locktype() lock.acquire() lock.release() del lock def test_try_acquire(self): lock = self.locktype() self.assertTrue(lock.acquire(False)) lock.release() def test_try_acquire_contended(self): lock = self.locktype() lock.acquire() result = [] def f(): result.append(lock.acquire(False)) Bunch(f, 1).wait_for_finished() self.assertFalse(result[0]) lock.release() def test_acquire_contended(self): lock = self.locktype() lock.acquire() N = 5 def f(): lock.acquire() lock.release() b = Bunch(f, N) b.wait_for_started() _wait() self.assertEqual(len(b.finished), 0) lock.release() b.wait_for_finished() self.assertEqual(len(b.finished), N) def test_with(self): lock = self.locktype() def f(): lock.acquire() lock.release() def _with(err=None): with lock: if err is not None: raise err _with() # Check the lock is unacquired Bunch(f, 1).wait_for_finished() self.assertRaises(TypeError, _with, TypeError) # Check the lock is unacquired Bunch(f, 1).wait_for_finished() def test_thread_leak(self): # The lock shouldn't leak a Thread instance when used from a foreign # (non-threading) thread. lock = self.locktype() def f(): lock.acquire() lock.release() n = len(threading.enumerate()) # We run many threads in the hope that existing threads ids won't # be recycled. Bunch(f, 15).wait_for_finished() self.assertEqual(n, len(threading.enumerate())) class LockTests(BaseLockTests): """ Tests for non-recursive, weak locks (which can be acquired and released from different threads). """ def test_reacquire(self): # Lock needs to be released before re-acquiring. lock = self.locktype() phase = [] def f(): lock.acquire() phase.append(None) lock.acquire() phase.append(None) start_new_thread(f, ()) while len(phase) == 0: _wait() _wait() self.assertEqual(len(phase), 1) lock.release() while len(phase) == 1: _wait() self.assertEqual(len(phase), 2) def test_different_thread(self): # Lock can be released from a different thread. lock = self.locktype() lock.acquire() def f(): lock.release() b = Bunch(f, 1) b.wait_for_finished() lock.acquire() lock.release() class RLockTests(BaseLockTests): """ Tests for recursive locks. """ def test_reacquire(self): lock = self.locktype() lock.acquire() lock.acquire() lock.release() lock.acquire() lock.release() lock.release() def test_release_unacquired(self): # Cannot release an unacquired lock lock = self.locktype() self.assertRaises(RuntimeError, lock.release) lock.acquire() lock.acquire() lock.release() lock.acquire() lock.release() lock.release() self.assertRaises(RuntimeError, lock.release) def test_different_thread(self): # Cannot release from a different thread lock = self.locktype() def f(): lock.acquire() b = Bunch(f, 1, True) try: self.assertRaises(RuntimeError, lock.release) finally: b.do_finish() def test__is_owned(self): lock = self.locktype() self.assertFalse(lock._is_owned()) lock.acquire() self.assertTrue(lock._is_owned()) lock.acquire() self.assertTrue(lock._is_owned()) result = [] def f(): result.append(lock._is_owned()) Bunch(f, 1).wait_for_finished() self.assertFalse(result[0]) lock.release() self.assertTrue(lock._is_owned()) lock.release() self.assertFalse(lock._is_owned()) class EventTests(BaseTestCase): """ Tests for Event objects. """ def test_is_set(self): evt = self.eventtype() self.assertFalse(evt.is_set()) evt.set() self.assertTrue(evt.is_set()) evt.set() self.assertTrue(evt.is_set()) evt.clear() self.assertFalse(evt.is_set()) evt.clear() self.assertFalse(evt.is_set()) def _check_notify(self, evt): # All threads get notified N = 5 results1 = [] results2 = [] def f(): results1.append(evt.wait()) results2.append(evt.wait()) b = Bunch(f, N) b.wait_for_started() _wait() self.assertEqual(len(results1), 0) evt.set() b.wait_for_finished() self.assertEqual(results1, [True] * N) self.assertEqual(results2, [True] * N) def test_notify(self): evt = self.eventtype() self._check_notify(evt) # Another time, after an explicit clear() evt.set() evt.clear() self._check_notify(evt) def test_timeout(self): evt = self.eventtype() results1 = [] results2 = [] N = 5 def f(): results1.append(evt.wait(0.0)) t1 = time.time() r = evt.wait(0.2) t2 = time.time() results2.append((r, t2 - t1)) Bunch(f, N).wait_for_finished() self.assertEqual(results1, [False] * N) for r, dt in results2: self.assertFalse(r) self.assertTrue(dt >= 0.2, dt) # The event is set results1 = [] results2 = [] evt.set() Bunch(f, N).wait_for_finished() self.assertEqual(results1, [True] * N) for r, dt in results2: self.assertTrue(r) @unittest.skip('grumpy') def test_reset_internal_locks(self): evt = self.eventtype() old_lock = evt._Event__cond._Condition__lock evt._reset_internal_locks() new_lock = evt._Event__cond._Condition__lock self.assertIsNot(new_lock, old_lock) self.assertIs(type(new_lock), type(old_lock)) class ConditionTests(BaseTestCase): """ Tests for condition variables. """ def test_acquire(self): cond = self.condtype() # Be default we have an RLock: the condition can be acquired multiple # times. cond.acquire() cond.acquire() cond.release() cond.release() lock = threading.Lock() cond = self.condtype(lock) cond.acquire() self.assertFalse(lock.acquire(False)) cond.release() self.assertTrue(lock.acquire(False)) self.assertFalse(cond.acquire(False)) lock.release() with cond: self.assertFalse(lock.acquire(False)) def test_unacquired_wait(self): cond = self.condtype() self.assertRaises(RuntimeError, cond.wait) def test_unacquired_notify(self): cond = self.condtype() self.assertRaises(RuntimeError, cond.notify) def _check_notify(self, cond): # Note that this test is sensitive to timing. If the worker threads # don't execute in a timely fashion, the main thread may think they # are further along then they are. The main thread therefore issues # _wait() statements to try to make sure that it doesn't race ahead # of the workers. # Secondly, this test assumes that condition variables are not subject # to spurious wakeups. The absence of spurious wakeups is an implementation # detail of Condition Cariables in current CPython, but in general, not # a guaranteed property of condition variables as a programming # construct. In particular, it is possible that this can no longer # be conveniently guaranteed should their implementation ever change. N = 5 ready = [] results1 = [] results2 = [] phase_num = 0 def f(): cond.acquire() ready.append(phase_num) cond.wait() cond.release() results1.append(phase_num) cond.acquire() ready.append(phase_num) cond.wait() cond.release() results2.append(phase_num) b = Bunch(f, N) b.wait_for_started() # first wait, to ensure all workers settle into cond.wait() before # we continue. See issues #8799 and #30727. while len(ready) < 5: _wait() ready = [] self.assertEqual(results1, []) # Notify 3 threads at first cond.acquire() cond.notify(3) _wait() phase_num = 1 cond.release() while len(results1) < 3: _wait() self.assertEqual(results1, [1] * 3) self.assertEqual(results2, []) # make sure all awaken workers settle into cond.wait() while len(ready) < 3: _wait() # Notify 5 threads: they might be in their first or second wait cond.acquire() cond.notify(5) _wait() phase_num = 2 cond.release() while len(results1) + len(results2) < 8: _wait() self.assertEqual(results1, [1] * 3 + [2] * 2) self.assertEqual(results2, [2] * 3) # make sure all workers settle into cond.wait() while len(ready) < 5: _wait() # Notify all threads: they are all in their second wait cond.acquire() cond.notify_all() _wait() phase_num = 3 cond.release() while len(results2) < 5: _wait() self.assertEqual(results1, [1] * 3 + [2] * 2) self.assertEqual(results2, [2] * 3 + [3] * 2) b.wait_for_finished() def test_notify(self): cond = self.condtype() self._check_notify(cond) # A second time, to check internal state is still ok. self._check_notify(cond) def test_timeout(self): cond = self.condtype() results = [] N = 5 def f(): cond.acquire() t1 = time.time() cond.wait(0.2) t2 = time.time() cond.release() results.append(t2 - t1) Bunch(f, N).wait_for_finished() self.assertEqual(len(results), 5) for dt in results: self.assertTrue(dt >= 0.2, dt) class BaseSemaphoreTests(BaseTestCase): """ Common tests for {bounded, unbounded} semaphore objects. """ def test_constructor(self): self.assertRaises(ValueError, self.semtype, value = -1) self.assertRaises(ValueError, self.semtype, value = -sys.maxint) def test_acquire(self): sem = self.semtype(1) sem.acquire() sem.release() sem = self.semtype(2) sem.acquire() sem.acquire() sem.release() sem.release() def test_acquire_destroy(self): sem = self.semtype() sem.acquire() del sem def test_acquire_contended(self): sem = self.semtype(7) sem.acquire() N = 10 results1 = [] results2 = [] phase_num = 0 def f(): sem.acquire() results1.append(phase_num) sem.acquire() results2.append(phase_num) b = Bunch(f, 10) b.wait_for_started() while len(results1) + len(results2) < 6: _wait() self.assertEqual(results1 + results2, [0] * 6) phase_num = 1 for i in range(7): sem.release() while len(results1) + len(results2) < 13: _wait() self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7) phase_num = 2 for i in range(6): sem.release() while len(results1) + len(results2) < 19: _wait() self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7 + [2] * 6) # The semaphore is still locked self.assertFalse(sem.acquire(False)) # Final release, to let the last thread finish sem.release() b.wait_for_finished() def test_try_acquire(self): sem = self.semtype(2) self.assertTrue(sem.acquire(False)) self.assertTrue(sem.acquire(False)) self.assertFalse(sem.acquire(False)) sem.release() self.assertTrue(sem.acquire(False)) def test_try_acquire_contended(self): sem = self.semtype(4) sem.acquire() results = [] def f(): results.append(sem.acquire(False)) results.append(sem.acquire(False)) Bunch(f, 5).wait_for_finished() # There can be a thread switch between acquiring the semaphore and # appending the result, therefore results will not necessarily be # ordered. self.assertEqual(sorted(results), [False] * 7 + [True] * 3 ) def test_default_value(self): # The default initial value is 1. sem = self.semtype() sem.acquire() def f(): sem.acquire() sem.release() b = Bunch(f, 1) b.wait_for_started() _wait() self.assertFalse(b.finished) sem.release() b.wait_for_finished() def test_with(self): sem = self.semtype(2) def _with(err=None): with sem: self.assertTrue(sem.acquire(False)) sem.release() with sem: self.assertFalse(sem.acquire(False)) if err: raise err _with() self.assertTrue(sem.acquire(False)) sem.release() self.assertRaises(TypeError, _with, TypeError) self.assertTrue(sem.acquire(False)) sem.release() class SemaphoreTests(BaseSemaphoreTests): """ Tests for unbounded semaphores. """ def test_release_unacquired(self): # Unbounded releases are allowed and increment the semaphore's value sem = self.semtype(1) sem.release() sem.acquire() sem.acquire() sem.release() class BoundedSemaphoreTests(BaseSemaphoreTests): """ Tests for bounded semaphores. """ def test_release_unacquired(self): # Cannot go past the initial value sem = self.semtype() self.assertRaises(ValueError, sem.release) sem.acquire() sem.release() self.assertRaises(ValueError, sem.release) ================================================ FILE: third_party/stdlib/test/mapping_tests.py ================================================ # tests common to dict and UserDict import unittest import UserDict from test import test_support class BasicTestMappingProtocol(unittest.TestCase): # This base class can be used to check that an object conforms to the # mapping protocol # Functions that can be useful to override to adapt to dictionary # semantics type2test = None # which class is being tested (overwrite in subclasses) def _reference(self): """Return a dictionary of values which are invariant by storage in the object under test.""" return {1:2, "key1":"value1", "key2":(1,2,3)} def _empty_mapping(self): """Return an empty mapping object""" return self.type2test() def _full_mapping(self, data): """Return a mapping object with the value contained in data dictionary""" x = self._empty_mapping() for key, value in data.items(): x[key] = value return x def __init__(self, *args, **kw): unittest.TestCase.__init__(self, *args, **kw) self.reference = self._reference().copy() # A (key, value) pair not in the mapping key, value = self.reference.popitem() self.other = {key:value} # A (key, value) pair in the mapping key, value = self.reference.popitem() self.inmapping = {key:value} self.reference[key] = value @unittest.skip('grumpy') def test_read(self): # Test for read only operations on mapping p = self._empty_mapping() p1 = dict(p) #workaround for singleton objects d = self._full_mapping(self.reference) if d is p: p = p1 #Indexing for key, value in self.reference.items(): self.assertEqual(d[key], value) knownkey = self.other.keys()[0] self.assertRaises(KeyError, lambda:d[knownkey]) #len self.assertEqual(len(p), 0) self.assertEqual(len(d), len(self.reference)) #in for k in self.reference: self.assertIn(k, d) for k in self.other: self.assertNotIn(k, d) #has_key with test_support.check_py3k_warnings(quiet=True): for k in self.reference: self.assertTrue(d.has_key(k)) for k in self.other: self.assertFalse(d.has_key(k)) #cmp self.assertEqual(cmp(p,p), 0) self.assertEqual(cmp(d,d), 0) self.assertEqual(cmp(p,d), -1) self.assertEqual(cmp(d,p), 1) #__non__zero__ if p: self.fail("Empty mapping must compare to False") if not d: self.fail("Full mapping must compare to True") # keys(), items(), iterkeys() ... def check_iterandlist(iter, lst, ref): self.assertTrue(hasattr(iter, 'next')) self.assertTrue(hasattr(iter, '__iter__')) x = list(iter) self.assertTrue(set(x)==set(lst)==set(ref)) check_iterandlist(d.iterkeys(), d.keys(), self.reference.keys()) check_iterandlist(iter(d), d.keys(), self.reference.keys()) check_iterandlist(d.itervalues(), d.values(), self.reference.values()) check_iterandlist(d.iteritems(), d.items(), self.reference.items()) #get key, value = d.iteritems().next() knownkey, knownvalue = self.other.iteritems().next() self.assertEqual(d.get(key, knownvalue), value) self.assertEqual(d.get(knownkey, knownvalue), knownvalue) self.assertNotIn(knownkey, d) def test_write(self): # Test for write operations on mapping p = self._empty_mapping() #Indexing for key, value in self.reference.items(): p[key] = value self.assertEqual(p[key], value) for key in self.reference.keys(): del p[key] self.assertRaises(KeyError, lambda:p[key]) p = self._empty_mapping() #update p.update(self.reference) self.assertEqual(dict(p), self.reference) items = p.items() p = self._empty_mapping() p.update(items) self.assertEqual(dict(p), self.reference) d = self._full_mapping(self.reference) #setdefault key, value = d.iteritems().next() knownkey, knownvalue = self.other.iteritems().next() self.assertEqual(d.setdefault(key, knownvalue), value) self.assertEqual(d[key], value) self.assertEqual(d.setdefault(knownkey, knownvalue), knownvalue) self.assertEqual(d[knownkey], knownvalue) #pop self.assertEqual(d.pop(knownkey), knownvalue) self.assertNotIn(knownkey, d) self.assertRaises(KeyError, d.pop, knownkey) default = 909 d[knownkey] = knownvalue self.assertEqual(d.pop(knownkey, default), knownvalue) self.assertNotIn(knownkey, d) self.assertEqual(d.pop(knownkey, default), default) #popitem key, value = d.popitem() self.assertNotIn(key, d) self.assertEqual(value, self.reference[key]) p=self._empty_mapping() self.assertRaises(KeyError, p.popitem) def test_constructor(self): self.assertEqual(self._empty_mapping(), self._empty_mapping()) def test_bool(self): self.assertTrue(not self._empty_mapping()) self.assertTrue(self.reference) self.assertTrue(bool(self._empty_mapping()) is False) self.assertTrue(bool(self.reference) is True) def test_keys(self): d = self._empty_mapping() self.assertEqual(d.keys(), []) d = self.reference self.assertIn(self.inmapping.keys()[0], d.keys()) self.assertNotIn(self.other.keys()[0], d.keys()) self.assertRaises(TypeError, d.keys, None) def test_values(self): d = self._empty_mapping() self.assertEqual(d.values(), []) self.assertRaises(TypeError, d.values, None) def test_items(self): d = self._empty_mapping() self.assertEqual(d.items(), []) self.assertRaises(TypeError, d.items, None) def test_len(self): d = self._empty_mapping() self.assertEqual(len(d), 0) def test_getitem(self): d = self.reference self.assertEqual(d[self.inmapping.keys()[0]], self.inmapping.values()[0]) self.assertRaises(TypeError, d.__getitem__) @unittest.skip('grumpy') def test_update(self): # mapping argument d = self._empty_mapping() d.update(self.other) self.assertEqual(d.items(), self.other.items()) # No argument d = self._empty_mapping() d.update() self.assertEqual(d, self._empty_mapping()) # item sequence d = self._empty_mapping() d.update(self.other.items()) self.assertEqual(d.items(), self.other.items()) # Iterator d = self._empty_mapping() d.update(self.other.iteritems()) self.assertEqual(d.items(), self.other.items()) # FIXME: Doesn't work with UserDict # self.assertRaises((TypeError, AttributeError), d.update, None) self.assertRaises((TypeError, AttributeError), d.update, 42) outerself = self class SimpleUserDict(object): def __init__(self): self.d = outerself.reference def keys(self): return self.d.keys() def __getitem__(self, i): return self.d[i] d.clear() d.update(SimpleUserDict()) i1 = d.items() i2 = self.reference.items() def safe_sort_key(kv): k, v = kv return id(type(k)), id(type(v)), k, v i1.sort(key=safe_sort_key) i2.sort(key=safe_sort_key) self.assertEqual(i1, i2) class Exc(Exception): pass d = self._empty_mapping() class FailingUserDict(object): def keys(self): raise Exc self.assertRaises(Exc, d.update, FailingUserDict()) d.clear() class FailingUserDict(object): def keys(self): class BogonIter(object): def __init__(self): self.i = 1 def __iter__(self): return self def next(self): if self.i: self.i = 0 return 'a' raise Exc return BogonIter() def __getitem__(self, key): return key self.assertRaises(Exc, d.update, FailingUserDict()) class FailingUserDict(object): def keys(self): class BogonIter(object): def __init__(self): self.i = ord('a') def __iter__(self): return self def next(self): if self.i <= ord('z'): rtn = chr(self.i) self.i += 1 return rtn raise StopIteration return BogonIter() def __getitem__(self, key): raise Exc self.assertRaises(Exc, d.update, FailingUserDict()) d = self._empty_mapping() class badseq(object): def __iter__(self): return self def next(self): raise Exc() self.assertRaises(Exc, d.update, badseq()) self.assertRaises(ValueError, d.update, [(1, 2, 3)]) # no test_fromkeys or test_copy as both os.environ and selves don't support it def test_get(self): d = self._empty_mapping() self.assertTrue(d.get(self.other.keys()[0]) is None) self.assertEqual(d.get(self.other.keys()[0], 3), 3) d = self.reference self.assertTrue(d.get(self.other.keys()[0]) is None) self.assertEqual(d.get(self.other.keys()[0], 3), 3) self.assertEqual(d.get(self.inmapping.keys()[0]), self.inmapping.values()[0]) self.assertEqual(d.get(self.inmapping.keys()[0], 3), self.inmapping.values()[0]) self.assertRaises(TypeError, d.get) self.assertRaises(TypeError, d.get, None, None, None) def test_setdefault(self): d = self._empty_mapping() self.assertRaises(TypeError, d.setdefault) def test_popitem(self): d = self._empty_mapping() self.assertRaises(KeyError, d.popitem) self.assertRaises(TypeError, d.popitem, 42) def test_pop(self): d = self._empty_mapping() k, v = self.inmapping.items()[0] d[k] = v self.assertRaises(KeyError, d.pop, self.other.keys()[0]) self.assertEqual(d.pop(k), v) self.assertEqual(len(d), 0) self.assertRaises(KeyError, d.pop, k) class TestMappingProtocol(BasicTestMappingProtocol): def test_constructor(self): BasicTestMappingProtocol.test_constructor(self) self.assertTrue(self._empty_mapping() is not self._empty_mapping()) self.assertEqual(self.type2test(x=1, y=2), {"x": 1, "y": 2}) def test_bool(self): BasicTestMappingProtocol.test_bool(self) self.assertTrue(not self._empty_mapping()) self.assertTrue(self._full_mapping({"x": "y"})) self.assertTrue(bool(self._empty_mapping()) is False) self.assertTrue(bool(self._full_mapping({"x": "y"})) is True) def test_keys(self): BasicTestMappingProtocol.test_keys(self) d = self._empty_mapping() self.assertEqual(d.keys(), []) d = self._full_mapping({'a': 1, 'b': 2}) k = d.keys() self.assertIn('a', k) self.assertIn('b', k) self.assertNotIn('c', k) def test_values(self): BasicTestMappingProtocol.test_values(self) d = self._full_mapping({1:2}) self.assertEqual(d.values(), [2]) def test_items(self): BasicTestMappingProtocol.test_items(self) d = self._full_mapping({1:2}) self.assertEqual(d.items(), [(1, 2)]) def test_has_key(self): d = self._empty_mapping() self.assertTrue(not d.has_key('a')) d = self._full_mapping({'a': 1, 'b': 2}) k = d.keys() k.sort(key=lambda k: (id(type(k)), k)) self.assertEqual(k, ['a', 'b']) self.assertRaises(TypeError, d.has_key) def test_contains(self): d = self._empty_mapping() self.assertNotIn('a', d) self.assertTrue(not ('a' in d)) self.assertTrue('a' not in d) d = self._full_mapping({'a': 1, 'b': 2}) self.assertIn('a', d) self.assertIn('b', d) self.assertNotIn('c', d) self.assertRaises(TypeError, d.__contains__) def test_len(self): BasicTestMappingProtocol.test_len(self) d = self._full_mapping({'a': 1, 'b': 2}) self.assertEqual(len(d), 2) def test_getitem(self): BasicTestMappingProtocol.test_getitem(self) d = self._full_mapping({'a': 1, 'b': 2}) self.assertEqual(d['a'], 1) self.assertEqual(d['b'], 2) d['c'] = 3 d['a'] = 4 self.assertEqual(d['c'], 3) self.assertEqual(d['a'], 4) del d['b'] self.assertEqual(d, {'a': 4, 'c': 3}) self.assertRaises(TypeError, d.__getitem__) def test_clear(self): d = self._full_mapping({1:1, 2:2, 3:3}) d.clear() self.assertEqual(d, {}) self.assertRaises(TypeError, d.clear, None) def test_update(self): BasicTestMappingProtocol.test_update(self) # mapping argument d = self._empty_mapping() d.update({1:100}) d.update({2:20}) d.update({1:1, 2:2, 3:3}) self.assertEqual(d, {1:1, 2:2, 3:3}) # no argument d.update() self.assertEqual(d, {1:1, 2:2, 3:3}) # keyword arguments d = self._empty_mapping() d.update(x=100) d.update(y=20) d.update(x=1, y=2, z=3) self.assertEqual(d, {"x":1, "y":2, "z":3}) # item sequence d = self._empty_mapping() d.update([("x", 100), ("y", 20)]) self.assertEqual(d, {"x":100, "y":20}) # Both item sequence and keyword arguments d = self._empty_mapping() d.update([("x", 100), ("y", 20)], x=1, y=2) self.assertEqual(d, {"x":1, "y":2}) # iterator d = self._full_mapping({1:3, 2:4}) d.update(self._full_mapping({1:2, 3:4, 5:6}).iteritems()) self.assertEqual(d, {1:2, 2:4, 3:4, 5:6}) class SimpleUserDict(object): def __init__(self): self.d = {1:1, 2:2, 3:3} def keys(self): return self.d.keys() def __getitem__(self, i): return self.d[i] d.clear() d.update(SimpleUserDict()) self.assertEqual(d, {1:1, 2:2, 3:3}) def test_fromkeys(self): self.assertEqual(self.type2test.fromkeys('abc'), {'a':None, 'b':None, 'c':None}) d = self._empty_mapping() self.assertTrue(not(d.fromkeys('abc') is d)) self.assertEqual(d.fromkeys('abc'), {'a':None, 'b':None, 'c':None}) self.assertEqual(d.fromkeys((4,5),0), {4:0, 5:0}) self.assertEqual(d.fromkeys([]), {}) def g(): yield 1 self.assertEqual(d.fromkeys(g()), {1:None}) self.assertRaises(TypeError, {}.fromkeys, 3) class dictlike(self.type2test): pass self.assertEqual(dictlike.fromkeys('a'), {'a':None}) self.assertEqual(dictlike().fromkeys('a'), {'a':None}) self.assertTrue(dictlike.fromkeys('a').__class__ is dictlike) self.assertTrue(dictlike().fromkeys('a').__class__ is dictlike) # FIXME: the following won't work with UserDict, because it's an old style class # self.assertTrue(type(dictlike.fromkeys('a')) is dictlike) class mydict(self.type2test): def __new__(cls): return UserDict.UserDict() ud = mydict.fromkeys('ab') self.assertEqual(ud, {'a':None, 'b':None}) # FIXME: the following won't work with UserDict, because it's an old style class # self.assertIsInstance(ud, UserDict.UserDict) self.assertRaises(TypeError, dict.fromkeys) class Exc(Exception): pass class baddict1(self.type2test): def __init__(self): raise Exc() self.assertRaises(Exc, baddict1.fromkeys, [1]) class BadSeq(object): def __iter__(self): return self def next(self): raise Exc() self.assertRaises(Exc, self.type2test.fromkeys, BadSeq()) class baddict2(self.type2test): def __setitem__(self, key, value): raise Exc() self.assertRaises(Exc, baddict2.fromkeys, [1]) def test_copy(self): d = self._full_mapping({1:1, 2:2, 3:3}) self.assertEqual(d.copy(), {1:1, 2:2, 3:3}) d = self._empty_mapping() self.assertEqual(d.copy(), d) self.assertIsInstance(d.copy(), d.__class__) self.assertRaises(TypeError, d.copy, None) def test_get(self): BasicTestMappingProtocol.test_get(self) d = self._empty_mapping() self.assertTrue(d.get('c') is None) self.assertEqual(d.get('c', 3), 3) d = self._full_mapping({'a' : 1, 'b' : 2}) self.assertTrue(d.get('c') is None) self.assertEqual(d.get('c', 3), 3) self.assertEqual(d.get('a'), 1) self.assertEqual(d.get('a', 3), 1) def test_setdefault(self): BasicTestMappingProtocol.test_setdefault(self) d = self._empty_mapping() self.assertTrue(d.setdefault('key0') is None) d.setdefault('key0', []) self.assertTrue(d.setdefault('key0') is None) d.setdefault('key', []).append(3) self.assertEqual(d['key'][0], 3) d.setdefault('key', []).append(4) self.assertEqual(len(d['key']), 2) def test_popitem(self): BasicTestMappingProtocol.test_popitem(self) # for copymode in -1, +1: for copymode in -1, 1: # -1: b has same structure as a # +1: b is a.copy() for log2size in range(12): size = 2**log2size a = self._empty_mapping() b = self._empty_mapping() for i in range(size): a[repr(i)] = i if copymode < 0: b[repr(i)] = i if copymode > 0: b = a.copy() for i in range(size): ka, va = ta = a.popitem() self.assertEqual(va, int(ka)) kb, vb = tb = b.popitem() self.assertEqual(vb, int(kb)) self.assertTrue(not(copymode < 0 and ta != tb)) self.assertTrue(not a) self.assertTrue(not b) def test_pop(self): BasicTestMappingProtocol.test_pop(self) # Tests for pop with specified key d = self._empty_mapping() k, v = 'abc', 'def' # verify longs/ints get same value when key > 32 bits (for 64-bit archs) # see SF bug #689659 x = 4503599627370496L y = 4503599627370496 h = self._full_mapping({x: 'anything', y: 'something else'}) self.assertEqual(h[x], h[y]) self.assertEqual(d.pop(k, v), v) d[k] = v self.assertEqual(d.pop(k, 1), v) class TestHashMappingProtocol(TestMappingProtocol): def test_getitem(self): TestMappingProtocol.test_getitem(self) class Exc(Exception): pass class BadEq(object): def __eq__(self, other): raise Exc() def __hash__(self): return 24 d = self._empty_mapping() d[BadEq()] = 42 self.assertRaises(KeyError, d.__getitem__, 23) class BadHash(object): fail = False def __hash__(self): if self.fail: raise Exc() else: return 42 d = self._empty_mapping() x = BadHash() d[x] = 42 x.fail = True self.assertRaises(Exc, d.__getitem__, x) def test_fromkeys(self): TestMappingProtocol.test_fromkeys(self) class mydict(self.type2test): def __new__(cls): return UserDict.UserDict() ud = mydict.fromkeys('ab') self.assertEqual(ud, {'a':None, 'b':None}) self.assertIsInstance(ud, UserDict.UserDict) def test_pop(self): TestMappingProtocol.test_pop(self) class Exc(Exception): pass class BadHash(object): fail = False def __hash__(self): if self.fail: raise Exc() else: return 42 d = self._empty_mapping() x = BadHash() d[x] = 42 x.fail = True self.assertRaises(Exc, d.pop, x) def test_mutatingiteration(self): d = self._empty_mapping() d[1] = 1 try: for i in d: d[i+1] = 1 except RuntimeError: pass else: self.fail("changing dict size during iteration doesn't raise Error") def test_repr(self): d = self._empty_mapping() self.assertEqual(repr(d), '{}') d[1] = 2 self.assertEqual(repr(d), '{1: 2}') d = self._empty_mapping() d[1] = d self.assertEqual(repr(d), '{1: {...}}') class Exc(Exception): pass class BadRepr(object): def __repr__(self): raise Exc() d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) def test_le(self): self.assertTrue(not (self._empty_mapping() < self._empty_mapping())) self.assertTrue(not (self._full_mapping({1: 2}) < self._full_mapping({1L: 2L}))) class Exc(Exception): pass class BadCmp(object): def __eq__(self, other): raise Exc() def __hash__(self): return 42 d1 = self._full_mapping({BadCmp(): 1}) d2 = self._full_mapping({1: 1}) try: d1 < d2 except Exc: pass else: self.fail("< didn't raise Exc") def test_setdefault(self): TestMappingProtocol.test_setdefault(self) class Exc(Exception): pass class BadHash(object): fail = False def __hash__(self): if self.fail: raise Exc() else: return 42 d = self._empty_mapping() x = BadHash() d[x] = 42 x.fail = True self.assertRaises(Exc, d.setdefault, x, []) ================================================ FILE: third_party/stdlib/test/seq_tests.py ================================================ """ Tests common to tuple, list and UserList.UserList """ import unittest import sys from test import test_support as support # Various iterables # This is used for checking the constructor (here and in test_deque.py) def iterfunc(seqn): 'Regular generator' for i in seqn: yield i class Sequence(object): 'Sequence using __getitem__' def __init__(self, seqn): self.seqn = seqn def __getitem__(self, i): return self.seqn[i] class IterFunc(object): 'Sequence using iterator protocol' def __init__(self, seqn): self.seqn = seqn self.i = 0 def __iter__(self): return self def next(self): if self.i >= len(self.seqn): raise StopIteration v = self.seqn[self.i] self.i += 1 return v class IterGen(object): 'Sequence using iterator protocol defined with a generator' def __init__(self, seqn): self.seqn = seqn self.i = 0 def __iter__(self): for val in self.seqn: yield val class IterNextOnly(object): 'Missing __getitem__ and __iter__' def __init__(self, seqn): self.seqn = seqn self.i = 0 def next(self): if self.i >= len(self.seqn): raise StopIteration v = self.seqn[self.i] self.i += 1 return v class IterNoNext(object): 'Iterator missing next()' def __init__(self, seqn): self.seqn = seqn self.i = 0 def __iter__(self): return self class IterGenExc(object): 'Test propagation of exceptions' def __init__(self, seqn): self.seqn = seqn self.i = 0 def __iter__(self): return self def next(self): 3 // 0 class IterFuncStop(object): 'Test immediate stop' def __init__(self, seqn): pass def __iter__(self): return self def next(self): raise StopIteration # from itertools import chain, imap # def itermulti(seqn): # 'Test multiple tiers of iterators' # return chain(imap(lambda x:x, iterfunc(IterGen(Sequence(seqn))))) class LyingTuple(tuple): def __iter__(self): yield 1 class LyingList(list): def __iter__(self): yield 1 class CommonTest(unittest.TestCase): # The type to be tested type2test = None def test_constructors(self): l0 = [] l1 = [0] l2 = [0, 1] u = self.type2test() u0 = self.type2test(l0) u1 = self.type2test(l1) u2 = self.type2test(l2) uu = self.type2test(u) uu0 = self.type2test(u0) uu1 = self.type2test(u1) uu2 = self.type2test(u2) v = self.type2test(tuple(u)) class OtherSeq(object): def __init__(self, initseq): self.__data = initseq def __len__(self): return len(self.__data) def __getitem__(self, i): return self.__data[i] s = OtherSeq(u0) v0 = self.type2test(s) self.assertEqual(len(v0), len(s)) s = "this is also a sequence" vv = self.type2test(s) self.assertEqual(len(vv), len(s)) # Create from various iteratables for s in ("123", "", range(1000), ('do', 1.2), xrange(2000,2200,5)): for g in (Sequence, IterFunc, IterGen, # itermulti, iterfunc): iterfunc): self.assertEqual(self.type2test(g(s)), self.type2test(s)) self.assertEqual(self.type2test(IterFuncStop(s)), self.type2test()) self.assertEqual(self.type2test(c for c in "123"), self.type2test("123")) self.assertRaises(TypeError, self.type2test, IterNextOnly(s)) self.assertRaises(TypeError, self.type2test, IterNoNext(s)) self.assertRaises(ZeroDivisionError, self.type2test, IterGenExc(s)) # Issue #23757 self.assertEqual(self.type2test(LyingTuple((2,))), self.type2test((1,))) self.assertEqual(self.type2test(LyingList([2])), self.type2test([1])) def test_truth(self): self.assertFalse(self.type2test()) self.assertTrue(self.type2test([42])) def test_getitem(self): u = self.type2test([0, 1, 2, 3, 4]) for i in xrange(len(u)): self.assertEqual(u[i], i) self.assertEqual(u[long(i)], i) for i in xrange(-len(u), -1): self.assertEqual(u[i], len(u)+i) self.assertEqual(u[long(i)], len(u)+i) self.assertRaises(IndexError, u.__getitem__, -len(u)-1) self.assertRaises(IndexError, u.__getitem__, len(u)) self.assertRaises(ValueError, u.__getitem__, slice(0,10,0)) u = self.type2test() self.assertRaises(IndexError, u.__getitem__, 0) self.assertRaises(IndexError, u.__getitem__, -1) self.assertRaises(TypeError, u.__getitem__) a = self.type2test([10, 11]) self.assertEqual(a[0], 10) self.assertEqual(a[1], 11) self.assertEqual(a[-2], 10) self.assertEqual(a[-1], 11) self.assertRaises(IndexError, a.__getitem__, -3) self.assertRaises(IndexError, a.__getitem__, 3) @unittest.skip('grumpy') def test_getslice(self): l = [0, 1, 2, 3, 4] u = self.type2test(l) self.assertEqual(u[0:0], self.type2test()) self.assertEqual(u[1:2], self.type2test([1])) self.assertEqual(u[-2:-1], self.type2test([3])) self.assertEqual(u[-1000:1000], u) self.assertEqual(u[1000:-1000], self.type2test([])) self.assertEqual(u[:], u) self.assertEqual(u[1:None], self.type2test([1, 2, 3, 4])) self.assertEqual(u[None:3], self.type2test([0, 1, 2])) # Extended slices self.assertEqual(u[::], u) self.assertEqual(u[::2], self.type2test([0, 2, 4])) self.assertEqual(u[1::2], self.type2test([1, 3])) self.assertEqual(u[::-1], self.type2test([4, 3, 2, 1, 0])) self.assertEqual(u[::-2], self.type2test([4, 2, 0])) self.assertEqual(u[3::-2], self.type2test([3, 1])) self.assertEqual(u[3:3:-2], self.type2test([])) self.assertEqual(u[3:2:-2], self.type2test([3])) self.assertEqual(u[3:1:-2], self.type2test([3])) self.assertEqual(u[3:0:-2], self.type2test([3, 1])) self.assertEqual(u[::-100], self.type2test([4])) self.assertEqual(u[100:-100:], self.type2test([])) self.assertEqual(u[-100:100:], u) self.assertEqual(u[100:-100:-1], u[::-1]) self.assertEqual(u[-100:100:-1], self.type2test([])) self.assertEqual(u[-100L:100L:2L], self.type2test([0, 2, 4])) # Test extreme cases with long ints a = self.type2test([0,1,2,3,4]) self.assertEqual(a[ -pow(2,128L): 3 ], self.type2test([0,1,2])) self.assertEqual(a[ 3: pow(2,145L) ], self.type2test([3,4])) self.assertRaises(TypeError, u.__getslice__) def test_contains(self): u = self.type2test([0, 1, 2]) for i in u: self.assertIn(i, u) for i in min(u)-1, max(u)+1: self.assertNotIn(i, u) self.assertRaises(TypeError, u.__contains__) def test_contains_fake(self): class AllEq(object): # Sequences must use rich comparison against each item # (unless "is" is true, or an earlier item answered) # So instances of AllEq must be found in all non-empty sequences. def __eq__(self, other): return True __hash__ = None # Can't meet hash invariant requirements self.assertNotIn(AllEq(), self.type2test([])) self.assertIn(AllEq(), self.type2test([1])) def test_contains_order(self): # Sequences must test in-order. If a rich comparison has side # effects, these will be visible to tests against later members. # In this test, the "side effect" is a short-circuiting raise. class DoNotTestEq(Exception): pass class StopCompares(object): def __eq__(self, other): raise DoNotTestEq checkfirst = self.type2test([1, StopCompares()]) self.assertIn(1, checkfirst) checklast = self.type2test([StopCompares(), 1]) self.assertRaises(DoNotTestEq, checklast.__contains__, 1) def test_len(self): self.assertEqual(len(self.type2test()), 0) self.assertEqual(len(self.type2test([])), 0) self.assertEqual(len(self.type2test([0])), 1) self.assertEqual(len(self.type2test([0, 1, 2])), 3) def test_minmax(self): u = self.type2test([0, 1, 2]) self.assertEqual(min(u), 0) self.assertEqual(max(u), 2) @unittest.skip('grumpy') def test_addmul(self): u1 = self.type2test([0]) u2 = self.type2test([0, 1]) self.assertEqual(u1, u1 + self.type2test()) self.assertEqual(u1, self.type2test() + u1) self.assertEqual(u1 + self.type2test([1]), u2) self.assertEqual(self.type2test([-1]) + u1, self.type2test([-1, 0])) self.assertEqual(self.type2test(), u2*0) self.assertEqual(self.type2test(), 0*u2) self.assertEqual(self.type2test(), u2*0L) self.assertEqual(self.type2test(), 0L*u2) self.assertEqual(u2, u2*1) self.assertEqual(u2, 1*u2) self.assertEqual(u2, u2*1L) self.assertEqual(u2, 1L*u2) self.assertEqual(u2+u2, u2*2) self.assertEqual(u2+u2, 2*u2) self.assertEqual(u2+u2, u2*2L) self.assertEqual(u2+u2, 2L*u2) self.assertEqual(u2+u2+u2, u2*3) self.assertEqual(u2+u2+u2, 3*u2) class subclass(self.type2test): pass u3 = subclass([0, 1]) self.assertEqual(u3, u3*1) self.assertIsNot(u3, u3*1) def test_iadd(self): u = self.type2test([0, 1]) u += self.type2test() self.assertEqual(u, self.type2test([0, 1])) u += self.type2test([2, 3]) self.assertEqual(u, self.type2test([0, 1, 2, 3])) u += self.type2test([4, 5]) self.assertEqual(u, self.type2test([0, 1, 2, 3, 4, 5])) u = self.type2test("spam") u += self.type2test("eggs") self.assertEqual(u, self.type2test("spameggs")) def test_imul(self): u = self.type2test([0, 1]) u *= 3 self.assertEqual(u, self.type2test([0, 1, 0, 1, 0, 1])) def test_getitemoverwriteiter(self): # Verify that __getitem__ overrides are not recognized by __iter__ class T(self.type2test): def __getitem__(self, key): return str(key) + '!!!' self.assertEqual(iter(T((1,2))).next(), 1) @unittest.skip('grumpy') def test_repeat(self): for m in xrange(4): s = tuple(range(m)) for n in xrange(-3, 5): self.assertEqual(self.type2test(s*n), self.type2test(s)*n) self.assertEqual(self.type2test(s)*(-4), self.type2test([])) self.assertEqual(id(s), id(s*1)) def test_bigrepeat(self): import sys if sys.maxint <= 2147483647: x = self.type2test([0]) # x *= 2**16 # self.assertRaises(MemoryError, x.__mul__, 2**16) # if hasattr(x, '__imul__'): # self.assertRaises(MemoryError, x.__imul__, 2**16) x *= 1<<16 self.assertRaises(MemoryError, x.__mul__, 1<<16) if hasattr(x, '__imul__'): self.assertRaises(MemoryError, x.__imul__, 1<<16) def test_subscript(self): a = self.type2test([10, 11]) self.assertEqual(a.__getitem__(0L), 10) self.assertEqual(a.__getitem__(1L), 11) self.assertEqual(a.__getitem__(-2L), 10) self.assertEqual(a.__getitem__(-1L), 11) self.assertRaises(IndexError, a.__getitem__, -3) self.assertRaises(IndexError, a.__getitem__, 3) self.assertEqual(a.__getitem__(slice(0,1)), self.type2test([10])) self.assertEqual(a.__getitem__(slice(1,2)), self.type2test([11])) self.assertEqual(a.__getitem__(slice(0,2)), self.type2test([10, 11])) self.assertEqual(a.__getitem__(slice(0,3)), self.type2test([10, 11])) self.assertEqual(a.__getitem__(slice(3,5)), self.type2test([])) self.assertRaises(ValueError, a.__getitem__, slice(0, 10, 0)) self.assertRaises(TypeError, a.__getitem__, 'x') def test_count(self): a = self.type2test([0, 1, 2])*3 self.assertEqual(a.count(0), 3) self.assertEqual(a.count(1), 3) self.assertEqual(a.count(3), 0) self.assertRaises(TypeError, a.count) class BadExc(Exception): pass class BadCmp(object): def __eq__(self, other): if other == 2: raise BadExc() return False self.assertRaises(BadExc, a.count, BadCmp()) @unittest.skip('grumpy') def test_index(self): u = self.type2test([0, 1]) self.assertEqual(u.index(0), 0) self.assertEqual(u.index(1), 1) self.assertRaises(ValueError, u.index, 2) u = self.type2test([-2, -1, 0, 0, 1, 2]) self.assertEqual(u.count(0), 2) self.assertEqual(u.index(0), 2) self.assertEqual(u.index(0, 2), 2) self.assertEqual(u.index(-2, -10), 0) self.assertEqual(u.index(0, 3), 3) self.assertEqual(u.index(0, 3, 4), 3) self.assertRaises(ValueError, u.index, 2, 0, -10) self.assertRaises(TypeError, u.index) class BadExc(Exception): pass class BadCmp(object): def __eq__(self, other): if other == 2: raise BadExc() return False a = self.type2test([0, 1, 2, 3]) self.assertRaises(BadExc, a.index, BadCmp()) a = self.type2test([-2, -1, 0, 0, 1, 2]) self.assertEqual(a.index(0), 2) self.assertEqual(a.index(0, 2), 2) self.assertEqual(a.index(0, -4), 2) self.assertEqual(a.index(-2, -10), 0) self.assertEqual(a.index(0, 3), 3) self.assertEqual(a.index(0, -3), 3) self.assertEqual(a.index(0, 3, 4), 3) self.assertEqual(a.index(0, -3, -2), 3) self.assertEqual(a.index(0, -4*sys.maxint, 4*sys.maxint), 2) self.assertRaises(ValueError, a.index, 0, 4*sys.maxint,-4*sys.maxint) self.assertRaises(ValueError, a.index, 2, 0, -10) @unittest.skip('grumpy') def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.type2test) support.check_free_after_iterating(self, reversed, self.type2test) ================================================ FILE: third_party/stdlib/test/string_tests.py ================================================ """ Common tests shared by test_str, test_unicode, test_userstring and test_string. """ # import unittest, string, sys, struct import unittest, string, sys import _struct as struct from test import test_support # from UserList import UserList import UserList as _UserList UserList = _UserList.UserList class Sequence(object): def __init__(self, seq='wxyz'): self.seq = seq def __len__(self): return len(self.seq) def __getitem__(self, i): return self.seq[i] class BadSeq1(Sequence): def __init__(self): self.seq = [7, 'hello', 123L] class BadSeq2(Sequence): def __init__(self): self.seq = ['a', 'b', 'c'] def __len__(self): return 8 class CommonTest(unittest.TestCase): # This testcase contains test that can be used in all # stringlike classes. Currently this is str, unicode # UserString and the string module. # The type to be tested # Change in subclasses to change the behaviour of fixtesttype() type2test = None # All tests pass their arguments to the testing methods # as str objects. fixtesttype() can be used to propagate # these arguments to the appropriate type def fixtype(self, obj): if isinstance(obj, str): return self.__class__.type2test(obj) elif isinstance(obj, list): return [self.fixtype(x) for x in obj] elif isinstance(obj, tuple): return tuple([self.fixtype(x) for x in obj]) elif isinstance(obj, dict): return dict([ (self.fixtype(key), self.fixtype(value)) for (key, value) in obj.iteritems() ]) else: return obj def test_fixtype(self): self.assertIs(type(self.fixtype("123")), self.type2test) # check that object.method(*args) returns result def checkequal(self, result, object, methodname, *args): result = self.fixtype(result) object = self.fixtype(object) args = self.fixtype(args) realresult = getattr(object, methodname)(*args) self.assertEqual( result, realresult ) # if the original is returned make sure that # this doesn't happen with subclasses if object == realresult: class subtype(self.__class__.type2test): pass object = subtype(object) realresult = getattr(object, methodname)(*args) self.assertTrue(object is not realresult) # check that object.method(*args) raises exc def checkraises(self, exc, obj, methodname, *args): obj = self.fixtype(obj) args = self.fixtype(args) with self.assertRaises(exc) as cm: getattr(obj, methodname)(*args) self.assertNotEqual(cm.exception.args[0], '') # call object.method(*args) without any checks def checkcall(self, object, methodname, *args): object = self.fixtype(object) args = self.fixtype(args) getattr(object, methodname)(*args) def test_hash(self): # SF bug 1054139: += optimization was not invalidating cached hash value a = self.type2test('DNSSEC') b = self.type2test('') for c in a: b += c hash(b) self.assertEqual(hash(a), hash(b)) # def test_capitalize(self): # self.checkequal(' hello ', ' hello ', 'capitalize') # self.checkequal('Hello ', 'Hello ','capitalize') # self.checkequal('Hello ', 'hello ','capitalize') # self.checkequal('Aaaa', 'aaaa', 'capitalize') # self.checkequal('Aaaa', 'AaAa', 'capitalize') # # self.checkraises(TypeError, 'hello', 'capitalize', 42) # def test_count(self): # self.checkequal(3, 'aaa', 'count', 'a') # self.checkequal(0, 'aaa', 'count', 'b') # self.checkequal(3, 'aaa', 'count', 'a') # self.checkequal(0, 'aaa', 'count', 'b') # self.checkequal(3, 'aaa', 'count', 'a') # self.checkequal(0, 'aaa', 'count', 'b') # self.checkequal(0, 'aaa', 'count', 'b') # self.checkequal(2, 'aaa', 'count', 'a', 1) # self.checkequal(0, 'aaa', 'count', 'a', 10) # self.checkequal(1, 'aaa', 'count', 'a', -1) # self.checkequal(3, 'aaa', 'count', 'a', -10) # self.checkequal(1, 'aaa', 'count', 'a', 0, 1) # self.checkequal(3, 'aaa', 'count', 'a', 0, 10) # self.checkequal(2, 'aaa', 'count', 'a', 0, -1) # self.checkequal(0, 'aaa', 'count', 'a', 0, -10) # self.checkequal(3, 'aaa', 'count', '', 1) # self.checkequal(1, 'aaa', 'count', '', 3) # self.checkequal(0, 'aaa', 'count', '', 10) # self.checkequal(2, 'aaa', 'count', '', -1) # self.checkequal(4, 'aaa', 'count', '', -10) # # self.checkequal(1, '', 'count', '') # self.checkequal(0, '', 'count', '', 1, 1) # self.checkequal(0, '', 'count', '', sys.maxint, 0) # # self.checkequal(0, '', 'count', 'xx') # self.checkequal(0, '', 'count', 'xx', 1, 1) # self.checkequal(0, '', 'count', 'xx', sys.maxint, 0) # # self.checkraises(TypeError, 'hello', 'count') # self.checkraises(TypeError, 'hello', 'count', 42) # # # For a variety of combinations, # # verify that str.count() matches an equivalent function # # replacing all occurrences and then differencing the string lengths # charset = ['', 'a', 'b'] # digits = 7 # base = len(charset) # teststrings = set() # for i in xrange(base ** digits): # entry = [] # for j in xrange(digits): # i, m = divmod(i, base) # entry.append(charset[m]) # teststrings.add(''.join(entry)) # teststrings = list(teststrings) # for i in teststrings: # i = self.fixtype(i) # n = len(i) # for j in teststrings: # r1 = i.count(j) # if j: # r2, rem = divmod(n - len(i.replace(j, '')), len(j)) # else: # r2, rem = len(i)+1, 0 # if rem or r1 != r2: # self.assertEqual(rem, 0, '%s != 0 for %s' % (rem, i)) # self.assertEqual(r1, r2, '%s != %s for %s' % (r1, r2, i)) # def test_find(self): # self.checkequal(0, 'abcdefghiabc', 'find', 'abc') # self.checkequal(9, 'abcdefghiabc', 'find', 'abc', 1) # self.checkequal(-1, 'abcdefghiabc', 'find', 'def', 4) # # self.checkequal(0, 'abc', 'find', '', 0) # self.checkequal(3, 'abc', 'find', '', 3) # self.checkequal(-1, 'abc', 'find', '', 4) # # # to check the ability to pass None as defaults # self.checkequal( 2, 'rrarrrrrrrrra', 'find', 'a') # self.checkequal(12, 'rrarrrrrrrrra', 'find', 'a', 4) # self.checkequal(-1, 'rrarrrrrrrrra', 'find', 'a', 4, 6) # self.checkequal(12, 'rrarrrrrrrrra', 'find', 'a', 4, None) # self.checkequal( 2, 'rrarrrrrrrrra', 'find', 'a', None, 6) # # self.checkraises(TypeError, 'hello', 'find') # self.checkraises(TypeError, 'hello', 'find', 42) # # self.checkequal(0, '', 'find', '') # self.checkequal(-1, '', 'find', '', 1, 1) # self.checkequal(-1, '', 'find', '', sys.maxint, 0) # # self.checkequal(-1, '', 'find', 'xx') # self.checkequal(-1, '', 'find', 'xx', 1, 1) # self.checkequal(-1, '', 'find', 'xx', sys.maxint, 0) # # # issue 7458 # self.checkequal(-1, 'ab', 'find', 'xxx', sys.maxsize + 1, 0) # # # For a variety of combinations, # # verify that str.find() matches __contains__ # # and that the found substring is really at that location # charset = ['', 'a', 'b', 'c'] # digits = 5 # base = len(charset) # teststrings = set() # for i in xrange(base ** digits): # entry = [] # for j in xrange(digits): # i, m = divmod(i, base) # entry.append(charset[m]) # teststrings.add(''.join(entry)) # teststrings = list(teststrings) # for i in teststrings: # i = self.fixtype(i) # for j in teststrings: # loc = i.find(j) # r1 = (loc != -1) # r2 = j in i # self.assertEqual(r1, r2) # if loc != -1: # self.assertEqual(i[loc:loc+len(j)], j) # def test_rfind(self): # self.checkequal(9, 'abcdefghiabc', 'rfind', 'abc') # self.checkequal(12, 'abcdefghiabc', 'rfind', '') # self.checkequal(0, 'abcdefghiabc', 'rfind', 'abcd') # self.checkequal(-1, 'abcdefghiabc', 'rfind', 'abcz') # # self.checkequal(3, 'abc', 'rfind', '', 0) # self.checkequal(3, 'abc', 'rfind', '', 3) # self.checkequal(-1, 'abc', 'rfind', '', 4) # # # to check the ability to pass None as defaults # self.checkequal(12, 'rrarrrrrrrrra', 'rfind', 'a') # self.checkequal(12, 'rrarrrrrrrrra', 'rfind', 'a', 4) # self.checkequal(-1, 'rrarrrrrrrrra', 'rfind', 'a', 4, 6) # self.checkequal(12, 'rrarrrrrrrrra', 'rfind', 'a', 4, None) # self.checkequal( 2, 'rrarrrrrrrrra', 'rfind', 'a', None, 6) # # self.checkraises(TypeError, 'hello', 'rfind') # self.checkraises(TypeError, 'hello', 'rfind', 42) # # # For a variety of combinations, # # verify that str.rfind() matches __contains__ # # and that the found substring is really at that location # charset = ['', 'a', 'b', 'c'] # digits = 5 # base = len(charset) # teststrings = set() # for i in xrange(base ** digits): # entry = [] # for j in xrange(digits): # i, m = divmod(i, base) # entry.append(charset[m]) # teststrings.add(''.join(entry)) # teststrings = list(teststrings) # for i in teststrings: # i = self.fixtype(i) # for j in teststrings: # loc = i.rfind(j) # r1 = (loc != -1) # r2 = j in i # self.assertEqual(r1, r2) # if loc != -1: # self.assertEqual(i[loc:loc+len(j)], self.fixtype(j)) # # # issue 7458 # self.checkequal(-1, 'ab', 'rfind', 'xxx', sys.maxsize + 1, 0) # def test_index(self): # self.checkequal(0, 'abcdefghiabc', 'index', '') # self.checkequal(3, 'abcdefghiabc', 'index', 'def') # self.checkequal(0, 'abcdefghiabc', 'index', 'abc') # self.checkequal(9, 'abcdefghiabc', 'index', 'abc', 1) # # self.checkraises(ValueError, 'abcdefghiabc', 'index', 'hib') # self.checkraises(ValueError, 'abcdefghiab', 'index', 'abc', 1) # self.checkraises(ValueError, 'abcdefghi', 'index', 'ghi', 8) # self.checkraises(ValueError, 'abcdefghi', 'index', 'ghi', -1) # # # to check the ability to pass None as defaults # self.checkequal( 2, 'rrarrrrrrrrra', 'index', 'a') # self.checkequal(12, 'rrarrrrrrrrra', 'index', 'a', 4) # self.checkraises(ValueError, 'rrarrrrrrrrra', 'index', 'a', 4, 6) # self.checkequal(12, 'rrarrrrrrrrra', 'index', 'a', 4, None) # self.checkequal( 2, 'rrarrrrrrrrra', 'index', 'a', None, 6) # # self.checkraises(TypeError, 'hello', 'index') # self.checkraises(TypeError, 'hello', 'index', 42) # def test_rindex(self): # self.checkequal(12, 'abcdefghiabc', 'rindex', '') # self.checkequal(3, 'abcdefghiabc', 'rindex', 'def') # self.checkequal(9, 'abcdefghiabc', 'rindex', 'abc') # self.checkequal(0, 'abcdefghiabc', 'rindex', 'abc', 0, -1) # # self.checkraises(ValueError, 'abcdefghiabc', 'rindex', 'hib') # self.checkraises(ValueError, 'defghiabc', 'rindex', 'def', 1) # self.checkraises(ValueError, 'defghiabc', 'rindex', 'abc', 0, -1) # self.checkraises(ValueError, 'abcdefghi', 'rindex', 'ghi', 0, 8) # self.checkraises(ValueError, 'abcdefghi', 'rindex', 'ghi', 0, -1) # # # to check the ability to pass None as defaults # self.checkequal(12, 'rrarrrrrrrrra', 'rindex', 'a') # self.checkequal(12, 'rrarrrrrrrrra', 'rindex', 'a', 4) # self.checkraises(ValueError, 'rrarrrrrrrrra', 'rindex', 'a', 4, 6) # self.checkequal(12, 'rrarrrrrrrrra', 'rindex', 'a', 4, None) # self.checkequal( 2, 'rrarrrrrrrrra', 'rindex', 'a', None, 6) # # self.checkraises(TypeError, 'hello', 'rindex') # self.checkraises(TypeError, 'hello', 'rindex', 42) # def test_lower(self): # self.checkequal('hello', 'HeLLo', 'lower') # self.checkequal('hello', 'hello', 'lower') # self.checkraises(TypeError, 'hello', 'lower', 42) # def test_upper(self): # self.checkequal('HELLO', 'HeLLo', 'upper') # self.checkequal('HELLO', 'HELLO', 'upper') # self.checkraises(TypeError, 'hello', 'upper', 42) # def test_expandtabs(self): # self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs') # self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs', 8) # self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs', 4) # self.checkequal('abc\r\nab def\ng hi', 'abc\r\nab\tdef\ng\thi', 'expandtabs', 4) # self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs') # self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs', 8) # self.checkequal('abc\r\nab\r\ndef\ng\r\nhi', 'abc\r\nab\r\ndef\ng\r\nhi', 'expandtabs', 4) # self.checkequal(' a\n b', ' \ta\n\tb', 'expandtabs', 1) # # self.checkraises(TypeError, 'hello', 'expandtabs', 42, 42) # # This test is only valid when sizeof(int) == sizeof(void*) == 4. # if sys.maxint < (1 << 32) and struct.calcsize('P') == 4: # self.checkraises(OverflowError, # '\ta\n\tb', 'expandtabs', sys.maxint) # def test_split(self): # self.checkequal(['this', 'is', 'the', 'split', 'function'], # 'this is the split function', 'split') # # # by whitespace # self.checkequal(['a', 'b', 'c', 'd'], 'a b c d ', 'split') # self.checkequal(['a', 'b c d'], 'a b c d', 'split', None, 1) # self.checkequal(['a', 'b', 'c d'], 'a b c d', 'split', None, 2) # self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'split', None, 3) # self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'split', None, 4) # self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'split', None, # sys.maxint-1) # self.checkequal(['a b c d'], 'a b c d', 'split', None, 0) # self.checkequal(['a b c d'], ' a b c d', 'split', None, 0) # self.checkequal(['a', 'b', 'c d'], 'a b c d', 'split', None, 2) # # self.checkequal([], ' ', 'split') # self.checkequal(['a'], ' a ', 'split') # self.checkequal(['a', 'b'], ' a b ', 'split') # self.checkequal(['a', 'b '], ' a b ', 'split', None, 1) # self.checkequal(['a b c '], ' a b c ', 'split', None, 0) # self.checkequal(['a', 'b c '], ' a b c ', 'split', None, 1) # self.checkequal(['a', 'b', 'c '], ' a b c ', 'split', None, 2) # self.checkequal(['a', 'b', 'c'], ' a b c ', 'split', None, 3) # self.checkequal(['a', 'b'], '\n\ta \t\r b \v ', 'split') # aaa = ' a '*20 # self.checkequal(['a']*20, aaa, 'split') # self.checkequal(['a'] + [aaa[4:]], aaa, 'split', None, 1) # self.checkequal(['a']*19 + ['a '], aaa, 'split', None, 19) # # for b in ('arf\tbarf', 'arf\nbarf', 'arf\rbarf', # 'arf\fbarf', 'arf\vbarf'): # self.checkequal(['arf', 'barf'], b, 'split') # self.checkequal(['arf', 'barf'], b, 'split', None) # self.checkequal(['arf', 'barf'], b, 'split', None, 2) # # # by a char # self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|') # self.checkequal(['a|b|c|d'], 'a|b|c|d', 'split', '|', 0) # self.checkequal(['a', 'b|c|d'], 'a|b|c|d', 'split', '|', 1) # self.checkequal(['a', 'b', 'c|d'], 'a|b|c|d', 'split', '|', 2) # self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|', 3) # self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|', 4) # self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|', # sys.maxint-2) # self.checkequal(['a|b|c|d'], 'a|b|c|d', 'split', '|', 0) # self.checkequal(['a', '', 'b||c||d'], 'a||b||c||d', 'split', '|', 2) # self.checkequal(['abcd'], 'abcd', 'split', '|') # self.checkequal([''], '', 'split', '|') # self.checkequal(['endcase ', ''], 'endcase |', 'split', '|') # self.checkequal(['', ' startcase'], '| startcase', 'split', '|') # self.checkequal(['', 'bothcase', ''], '|bothcase|', 'split', '|') # self.checkequal(['a', '', 'b\x00c\x00d'], 'a\x00\x00b\x00c\x00d', 'split', '\x00', 2) # # self.checkequal(['a']*20, ('a|'*20)[:-1], 'split', '|') # self.checkequal(['a']*15 +['a|a|a|a|a'], # ('a|'*20)[:-1], 'split', '|', 15) # # # by string # self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//') # self.checkequal(['a', 'b//c//d'], 'a//b//c//d', 'split', '//', 1) # self.checkequal(['a', 'b', 'c//d'], 'a//b//c//d', 'split', '//', 2) # self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//', 3) # self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//', 4) # self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//', # sys.maxint-10) # self.checkequal(['a//b//c//d'], 'a//b//c//d', 'split', '//', 0) # self.checkequal(['a', '', 'b////c////d'], 'a////b////c////d', 'split', '//', 2) # self.checkequal(['endcase ', ''], 'endcase test', 'split', 'test') # self.checkequal(['', ' begincase'], 'test begincase', 'split', 'test') # self.checkequal(['', ' bothcase ', ''], 'test bothcase test', # 'split', 'test') # self.checkequal(['a', 'bc'], 'abbbc', 'split', 'bb') # self.checkequal(['', ''], 'aaa', 'split', 'aaa') # self.checkequal(['aaa'], 'aaa', 'split', 'aaa', 0) # self.checkequal(['ab', 'ab'], 'abbaab', 'split', 'ba') # self.checkequal(['aaaa'], 'aaaa', 'split', 'aab') # self.checkequal([''], '', 'split', 'aaa') # self.checkequal(['aa'], 'aa', 'split', 'aaa') # self.checkequal(['A', 'bobb'], 'Abbobbbobb', 'split', 'bbobb') # self.checkequal(['A', 'B', ''], 'AbbobbBbbobb', 'split', 'bbobb') # # self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'split', 'BLAH') # self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'split', 'BLAH', 19) # self.checkequal(['a']*18 + ['aBLAHa'], ('aBLAH'*20)[:-4], # 'split', 'BLAH', 18) # # # mixed use of str and unicode # if self.type2test is not bytearray: # result = [u'a', u'b', u'c d'] # self.checkequal(result, 'a b c d', 'split', u' ', 2) # # # argument type # self.checkraises(TypeError, 'hello', 'split', 42, 42, 42) # # # null case # self.checkraises(ValueError, 'hello', 'split', '') # self.checkraises(ValueError, 'hello', 'split', '', 0) # def test_rsplit(self): # self.checkequal(['this', 'is', 'the', 'rsplit', 'function'], # 'this is the rsplit function', 'rsplit') # # # by whitespace # self.checkequal(['a', 'b', 'c', 'd'], 'a b c d ', 'rsplit') # self.checkequal(['a b c', 'd'], 'a b c d', 'rsplit', None, 1) # self.checkequal(['a b', 'c', 'd'], 'a b c d', 'rsplit', None, 2) # self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', None, 3) # self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', None, 4) # self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', None, # sys.maxint-20) # self.checkequal(['a b c d'], 'a b c d', 'rsplit', None, 0) # self.checkequal(['a b c d'], 'a b c d ', 'rsplit', None, 0) # self.checkequal(['a b', 'c', 'd'], 'a b c d', 'rsplit', None, 2) # # self.checkequal([], ' ', 'rsplit') # self.checkequal(['a'], ' a ', 'rsplit') # self.checkequal(['a', 'b'], ' a b ', 'rsplit') # self.checkequal([' a', 'b'], ' a b ', 'rsplit', None, 1) # self.checkequal([' a b c'], ' a b c ', 'rsplit', # None, 0) # self.checkequal([' a b','c'], ' a b c ', 'rsplit', # None, 1) # self.checkequal([' a', 'b', 'c'], ' a b c ', 'rsplit', # None, 2) # self.checkequal(['a', 'b', 'c'], ' a b c ', 'rsplit', # None, 3) # self.checkequal(['a', 'b'], '\n\ta \t\r b \v ', 'rsplit', None, 88) # aaa = ' a '*20 # self.checkequal(['a']*20, aaa, 'rsplit') # self.checkequal([aaa[:-4]] + ['a'], aaa, 'rsplit', None, 1) # self.checkequal([' a a'] + ['a']*18, aaa, 'rsplit', None, 18) # # for b in ('arf\tbarf', 'arf\nbarf', 'arf\rbarf', # 'arf\fbarf', 'arf\vbarf'): # self.checkequal(['arf', 'barf'], b, 'rsplit') # self.checkequal(['arf', 'barf'], b, 'rsplit', None) # self.checkequal(['arf', 'barf'], b, 'rsplit', None, 2) # # # by a char # self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|') # self.checkequal(['a|b|c', 'd'], 'a|b|c|d', 'rsplit', '|', 1) # self.checkequal(['a|b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', 2) # self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', 3) # self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', 4) # self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', # sys.maxint-100) # self.checkequal(['a|b|c|d'], 'a|b|c|d', 'rsplit', '|', 0) # self.checkequal(['a||b||c', '', 'd'], 'a||b||c||d', 'rsplit', '|', 2) # self.checkequal(['abcd'], 'abcd', 'rsplit', '|') # self.checkequal([''], '', 'rsplit', '|') # self.checkequal(['', ' begincase'], '| begincase', 'rsplit', '|') # self.checkequal(['endcase ', ''], 'endcase |', 'rsplit', '|') # self.checkequal(['', 'bothcase', ''], '|bothcase|', 'rsplit', '|') # # self.checkequal(['a\x00\x00b', 'c', 'd'], 'a\x00\x00b\x00c\x00d', 'rsplit', '\x00', 2) # # self.checkequal(['a']*20, ('a|'*20)[:-1], 'rsplit', '|') # self.checkequal(['a|a|a|a|a']+['a']*15, # ('a|'*20)[:-1], 'rsplit', '|', 15) # # # by string # self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//') # self.checkequal(['a//b//c', 'd'], 'a//b//c//d', 'rsplit', '//', 1) # self.checkequal(['a//b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', 2) # self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', 3) # self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', 4) # self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', # sys.maxint-5) # self.checkequal(['a//b//c//d'], 'a//b//c//d', 'rsplit', '//', 0) # self.checkequal(['a////b////c', '', 'd'], 'a////b////c////d', 'rsplit', '//', 2) # self.checkequal(['', ' begincase'], 'test begincase', 'rsplit', 'test') # self.checkequal(['endcase ', ''], 'endcase test', 'rsplit', 'test') # self.checkequal(['', ' bothcase ', ''], 'test bothcase test', # 'rsplit', 'test') # self.checkequal(['ab', 'c'], 'abbbc', 'rsplit', 'bb') # self.checkequal(['', ''], 'aaa', 'rsplit', 'aaa') # self.checkequal(['aaa'], 'aaa', 'rsplit', 'aaa', 0) # self.checkequal(['ab', 'ab'], 'abbaab', 'rsplit', 'ba') # self.checkequal(['aaaa'], 'aaaa', 'rsplit', 'aab') # self.checkequal([''], '', 'rsplit', 'aaa') # self.checkequal(['aa'], 'aa', 'rsplit', 'aaa') # self.checkequal(['bbob', 'A'], 'bbobbbobbA', 'rsplit', 'bbobb') # self.checkequal(['', 'B', 'A'], 'bbobbBbbobbA', 'rsplit', 'bbobb') # # self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'rsplit', 'BLAH') # self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'rsplit', 'BLAH', 19) # self.checkequal(['aBLAHa'] + ['a']*18, ('aBLAH'*20)[:-4], # 'rsplit', 'BLAH', 18) # # # mixed use of str and unicode # if self.type2test is not bytearray: # result = [u'a b', u'c', u'd'] # self.checkequal(result, 'a b c d', 'rsplit', u' ', 2) # # # argument type # self.checkraises(TypeError, 'hello', 'rsplit', 42, 42, 42) # # # null case # self.checkraises(ValueError, 'hello', 'rsplit', '') # self.checkraises(ValueError, 'hello', 'rsplit', '', 0) def test_strip_whitespace(self): self.checkequal('hello', ' hello ', 'strip') self.checkequal('hello ', ' hello ', 'lstrip') self.checkequal(' hello', ' hello ', 'rstrip') self.checkequal('hello', 'hello', 'strip') b = ' \t\n\r\f\vabc \t\n\r\f\v' self.checkequal('abc', b, 'strip') self.checkequal('abc \t\n\r\f\v', b, 'lstrip') self.checkequal(' \t\n\r\f\vabc', b, 'rstrip') # strip/lstrip/rstrip with None arg self.checkequal('hello', ' hello ', 'strip', None) self.checkequal('hello ', ' hello ', 'lstrip', None) self.checkequal(' hello', ' hello ', 'rstrip', None) self.checkequal('hello', 'hello', 'strip', None) # def test_strip(self): # # strip/lstrip/rstrip with str arg # self.checkequal('hello', 'xyzzyhelloxyzzy', 'strip', 'xyz') # self.checkequal('helloxyzzy', 'xyzzyhelloxyzzy', 'lstrip', 'xyz') # self.checkequal('xyzzyhello', 'xyzzyhelloxyzzy', 'rstrip', 'xyz') # self.checkequal('hello', 'hello', 'strip', 'xyz') # self.checkequal('', 'mississippi', 'strip', 'mississippi') # # # only trims the start and end, does not strip internal characters # self.checkequal('mississipp', 'mississippi', 'strip', 'i') # # # strip/lstrip/rstrip with unicode arg # if self.type2test is not bytearray and test_support.have_unicode: # self.checkequal(unicode('hello', 'ascii'), 'xyzzyhelloxyzzy', # 'strip', unicode('xyz', 'ascii')) # self.checkequal(unicode('helloxyzzy', 'ascii'), 'xyzzyhelloxyzzy', # 'lstrip', unicode('xyz', 'ascii')) # self.checkequal(unicode('xyzzyhello', 'ascii'), 'xyzzyhelloxyzzy', # 'rstrip', unicode('xyz', 'ascii')) # # XXX # #self.checkequal(unicode('hello', 'ascii'), 'hello', # # 'strip', unicode('xyz', 'ascii')) # # self.checkraises(TypeError, 'hello', 'strip', 42, 42) # self.checkraises(TypeError, 'hello', 'lstrip', 42, 42) # self.checkraises(TypeError, 'hello', 'rstrip', 42, 42) # def test_ljust(self): # self.checkequal('abc ', 'abc', 'ljust', 10) # self.checkequal('abc ', 'abc', 'ljust', 6) # self.checkequal('abc', 'abc', 'ljust', 3) # self.checkequal('abc', 'abc', 'ljust', 2) # if self.type2test is bytearray: # # Special case because bytearray argument is not accepted # self.assertEqual(b'abc*******', bytearray(b'abc').ljust(10, '*')) # else: # self.checkequal('abc*******', 'abc', 'ljust', 10, '*') # self.checkraises(TypeError, 'abc', 'ljust') # def test_rjust(self): # self.checkequal(' abc', 'abc', 'rjust', 10) # self.checkequal(' abc', 'abc', 'rjust', 6) # self.checkequal('abc', 'abc', 'rjust', 3) # self.checkequal('abc', 'abc', 'rjust', 2) # if self.type2test is bytearray: # # Special case because bytearray argument is not accepted # self.assertEqual(b'*******abc', bytearray(b'abc').rjust(10, '*')) # else: # self.checkequal('*******abc', 'abc', 'rjust', 10, '*') # self.checkraises(TypeError, 'abc', 'rjust') # def test_center(self): # self.checkequal(' abc ', 'abc', 'center', 10) # self.checkequal(' abc ', 'abc', 'center', 6) # self.checkequal('abc', 'abc', 'center', 3) # self.checkequal('abc', 'abc', 'center', 2) # if self.type2test is bytearray: # # Special case because bytearray argument is not accepted # result = bytearray(b'abc').center(10, '*') # self.assertEqual(b'***abc****', result) # else: # self.checkequal('***abc****', 'abc', 'center', 10, '*') # self.checkraises(TypeError, 'abc', 'center') # def test_swapcase(self): # self.checkequal('hEllO CoMPuTErS', 'HeLLo cOmpUteRs', 'swapcase') # # self.checkraises(TypeError, 'hello', 'swapcase', 42) # def test_replace(self): # EQ = self.checkequal # # # Operations on the empty string # EQ("", "", "replace", "", "") # EQ("A", "", "replace", "", "A") # EQ("", "", "replace", "A", "") # EQ("", "", "replace", "A", "A") # EQ("", "", "replace", "", "", 100) # EQ("", "", "replace", "", "", sys.maxint) # # # interleave (from=="", 'to' gets inserted everywhere) # EQ("A", "A", "replace", "", "") # EQ("*A*", "A", "replace", "", "*") # EQ("*1A*1", "A", "replace", "", "*1") # EQ("*-#A*-#", "A", "replace", "", "*-#") # EQ("*-A*-A*-", "AA", "replace", "", "*-") # EQ("*-A*-A*-", "AA", "replace", "", "*-", -1) # EQ("*-A*-A*-", "AA", "replace", "", "*-", sys.maxint) # EQ("*-A*-A*-", "AA", "replace", "", "*-", 4) # EQ("*-A*-A*-", "AA", "replace", "", "*-", 3) # EQ("*-A*-A", "AA", "replace", "", "*-", 2) # EQ("*-AA", "AA", "replace", "", "*-", 1) # EQ("AA", "AA", "replace", "", "*-", 0) # # # single character deletion (from=="A", to=="") # EQ("", "A", "replace", "A", "") # EQ("", "AAA", "replace", "A", "") # EQ("", "AAA", "replace", "A", "", -1) # EQ("", "AAA", "replace", "A", "", sys.maxint) # EQ("", "AAA", "replace", "A", "", 4) # EQ("", "AAA", "replace", "A", "", 3) # EQ("A", "AAA", "replace", "A", "", 2) # EQ("AA", "AAA", "replace", "A", "", 1) # EQ("AAA", "AAA", "replace", "A", "", 0) # EQ("", "AAAAAAAAAA", "replace", "A", "") # EQ("BCD", "ABACADA", "replace", "A", "") # EQ("BCD", "ABACADA", "replace", "A", "", -1) # EQ("BCD", "ABACADA", "replace", "A", "", sys.maxint) # EQ("BCD", "ABACADA", "replace", "A", "", 5) # EQ("BCD", "ABACADA", "replace", "A", "", 4) # EQ("BCDA", "ABACADA", "replace", "A", "", 3) # EQ("BCADA", "ABACADA", "replace", "A", "", 2) # EQ("BACADA", "ABACADA", "replace", "A", "", 1) # EQ("ABACADA", "ABACADA", "replace", "A", "", 0) # EQ("BCD", "ABCAD", "replace", "A", "") # EQ("BCD", "ABCADAA", "replace", "A", "") # EQ("BCD", "BCD", "replace", "A", "") # EQ("*************", "*************", "replace", "A", "") # EQ("^A^", "^"+"A"*1000+"^", "replace", "A", "", 999) # # # substring deletion (from=="the", to=="") # EQ("", "the", "replace", "the", "") # EQ("ater", "theater", "replace", "the", "") # EQ("", "thethe", "replace", "the", "") # EQ("", "thethethethe", "replace", "the", "") # EQ("aaaa", "theatheatheathea", "replace", "the", "") # EQ("that", "that", "replace", "the", "") # EQ("thaet", "thaet", "replace", "the", "") # EQ("here and re", "here and there", "replace", "the", "") # EQ("here and re and re", "here and there and there", # "replace", "the", "", sys.maxint) # EQ("here and re and re", "here and there and there", # "replace", "the", "", -1) # EQ("here and re and re", "here and there and there", # "replace", "the", "", 3) # EQ("here and re and re", "here and there and there", # "replace", "the", "", 2) # EQ("here and re and there", "here and there and there", # "replace", "the", "", 1) # EQ("here and there and there", "here and there and there", # "replace", "the", "", 0) # EQ("here and re and re", "here and there and there", "replace", "the", "") # # EQ("abc", "abc", "replace", "the", "") # EQ("abcdefg", "abcdefg", "replace", "the", "") # # # substring deletion (from=="bob", to=="") # EQ("bob", "bbobob", "replace", "bob", "") # EQ("bobXbob", "bbobobXbbobob", "replace", "bob", "") # EQ("aaaaaaa", "aaaaaaabob", "replace", "bob", "") # EQ("aaaaaaa", "aaaaaaa", "replace", "bob", "") # # # single character replace in place (len(from)==len(to)==1) # EQ("Who goes there?", "Who goes there?", "replace", "o", "o") # EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O") # EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", sys.maxint) # EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", -1) # EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", 3) # EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", 2) # EQ("WhO goes there?", "Who goes there?", "replace", "o", "O", 1) # EQ("Who goes there?", "Who goes there?", "replace", "o", "O", 0) # # EQ("Who goes there?", "Who goes there?", "replace", "a", "q") # EQ("who goes there?", "Who goes there?", "replace", "W", "w") # EQ("wwho goes there?ww", "WWho goes there?WW", "replace", "W", "w") # EQ("Who goes there!", "Who goes there?", "replace", "?", "!") # EQ("Who goes there!!", "Who goes there??", "replace", "?", "!") # # EQ("Who goes there?", "Who goes there?", "replace", ".", "!") # # # substring replace in place (len(from)==len(to) > 1) # EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**") # EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", sys.maxint) # EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", -1) # EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", 4) # EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", 3) # EQ("Th** ** a tissue", "This is a tissue", "replace", "is", "**", 2) # EQ("Th** is a tissue", "This is a tissue", "replace", "is", "**", 1) # EQ("This is a tissue", "This is a tissue", "replace", "is", "**", 0) # EQ("cobob", "bobob", "replace", "bob", "cob") # EQ("cobobXcobocob", "bobobXbobobob", "replace", "bob", "cob") # EQ("bobob", "bobob", "replace", "bot", "bot") # # # replace single character (len(from)==1, len(to)>1) # EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK") # EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK", -1) # EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK", sys.maxint) # EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK", 2) # EQ("ReyKKjavik", "Reykjavik", "replace", "k", "KK", 1) # EQ("Reykjavik", "Reykjavik", "replace", "k", "KK", 0) # EQ("A----B----C----", "A.B.C.", "replace", ".", "----") # # EQ("Reykjavik", "Reykjavik", "replace", "q", "KK") # # # replace substring (len(from)>1, len(to)!=len(from)) # EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", # "replace", "spam", "ham") # EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", # "replace", "spam", "ham", sys.maxint) # EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", # "replace", "spam", "ham", -1) # EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", # "replace", "spam", "ham", 4) # EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam", # "replace", "spam", "ham", 3) # EQ("ham, ham, eggs and spam", "spam, spam, eggs and spam", # "replace", "spam", "ham", 2) # EQ("ham, spam, eggs and spam", "spam, spam, eggs and spam", # "replace", "spam", "ham", 1) # EQ("spam, spam, eggs and spam", "spam, spam, eggs and spam", # "replace", "spam", "ham", 0) # # EQ("bobob", "bobobob", "replace", "bobob", "bob") # EQ("bobobXbobob", "bobobobXbobobob", "replace", "bobob", "bob") # EQ("BOBOBOB", "BOBOBOB", "replace", "bob", "bobby") # # with test_support.check_py3k_warnings(): # ba = buffer('a') # bb = buffer('b') # EQ("bbc", "abc", "replace", ba, bb) # EQ("aac", "abc", "replace", bb, ba) # # # # self.checkequal('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1) # self.checkequal('onetwothree', 'one!two!three!', 'replace', '!', '') # self.checkequal('one@two@three!', 'one!two!three!', 'replace', '!', '@', 2) # self.checkequal('one@two@three@', 'one!two!three!', 'replace', '!', '@', 3) # self.checkequal('one@two@three@', 'one!two!three!', 'replace', '!', '@', 4) # self.checkequal('one!two!three!', 'one!two!three!', 'replace', '!', '@', 0) # self.checkequal('one@two@three@', 'one!two!three!', 'replace', '!', '@') # self.checkequal('one!two!three!', 'one!two!three!', 'replace', 'x', '@') # self.checkequal('one!two!three!', 'one!two!three!', 'replace', 'x', '@', 2) # self.checkequal('-a-b-c-', 'abc', 'replace', '', '-') # self.checkequal('-a-b-c', 'abc', 'replace', '', '-', 3) # self.checkequal('abc', 'abc', 'replace', '', '-', 0) # self.checkequal('', '', 'replace', '', '') # self.checkequal('abc', 'abc', 'replace', 'ab', '--', 0) # self.checkequal('abc', 'abc', 'replace', 'xy', '--') # # Next three for SF bug 422088: [OSF1 alpha] string.replace(); died with # # MemoryError due to empty result (platform malloc issue when requesting # # 0 bytes). # self.checkequal('', '123', 'replace', '123', '') # self.checkequal('', '123123', 'replace', '123', '') # self.checkequal('x', '123x123', 'replace', '123', '') # # self.checkraises(TypeError, 'hello', 'replace') # self.checkraises(TypeError, 'hello', 'replace', 42) # self.checkraises(TypeError, 'hello', 'replace', 42, 'h') # self.checkraises(TypeError, 'hello', 'replace', 'h', 42) @unittest.skipIf(sys.maxint > (1 << 32) or struct.calcsize('P') != 4, 'only applies to 32-bit platforms') def test_replace_overflow(self): # Check for overflow checking on 32 bit machines A2_16 = "A" * (2**16) self.checkraises(OverflowError, A2_16, "replace", "", A2_16) self.checkraises(OverflowError, A2_16, "replace", "A", A2_16) self.checkraises(OverflowError, A2_16, "replace", "AA", A2_16+A2_16) # def test_zfill(self): # self.checkequal('123', '123', 'zfill', 2) # self.checkequal('123', '123', 'zfill', 3) # self.checkequal('0123', '123', 'zfill', 4) # self.checkequal('+123', '+123', 'zfill', 3) # self.checkequal('+123', '+123', 'zfill', 4) # self.checkequal('+0123', '+123', 'zfill', 5) # self.checkequal('-123', '-123', 'zfill', 3) # self.checkequal('-123', '-123', 'zfill', 4) # self.checkequal('-0123', '-123', 'zfill', 5) # self.checkequal('000', '', 'zfill', 3) # self.checkequal('34', '34', 'zfill', 1) # self.checkequal('0034', '34', 'zfill', 4) # # self.checkraises(TypeError, '123', 'zfill') class NonStringModuleTest(object): # additional test cases for all string classes from bytearray to # UserString, but not valid for the "string" module def test_islower(self): self.checkequal(False, '', 'islower') self.checkequal(True, 'a', 'islower') self.checkequal(False, 'A', 'islower') self.checkequal(False, '\n', 'islower') self.checkequal(True, 'abc', 'islower') self.checkequal(False, 'aBc', 'islower') self.checkequal(True, 'abc\n', 'islower') self.checkraises(TypeError, 'abc', 'islower', 42) def test_isupper(self): self.checkequal(False, '', 'isupper') self.checkequal(False, 'a', 'isupper') self.checkequal(True, 'A', 'isupper') self.checkequal(False, '\n', 'isupper') self.checkequal(True, 'ABC', 'isupper') self.checkequal(False, 'AbC', 'isupper') self.checkequal(True, 'ABC\n', 'isupper') self.checkraises(TypeError, 'abc', 'isupper', 42) def test_istitle(self): self.checkequal(False, '', 'istitle') self.checkequal(False, 'a', 'istitle') self.checkequal(True, 'A', 'istitle') self.checkequal(False, '\n', 'istitle') self.checkequal(True, 'A Titlecased Line', 'istitle') self.checkequal(True, 'A\nTitlecased Line', 'istitle') self.checkequal(True, 'A Titlecased, Line', 'istitle') self.checkequal(False, 'Not a capitalized String', 'istitle') self.checkequal(False, 'Not\ta Titlecase String', 'istitle') self.checkequal(False, 'Not--a Titlecase String', 'istitle') self.checkequal(False, 'NOT', 'istitle') self.checkraises(TypeError, 'abc', 'istitle', 42) def test_isspace(self): self.checkequal(False, '', 'isspace') self.checkequal(False, 'a', 'isspace') self.checkequal(True, ' ', 'isspace') self.checkequal(True, '\t', 'isspace') self.checkequal(True, '\r', 'isspace') self.checkequal(True, '\n', 'isspace') self.checkequal(True, ' \t\r\n', 'isspace') self.checkequal(False, ' \t\r\na', 'isspace') self.checkraises(TypeError, 'abc', 'isspace', 42) def test_isalpha(self): self.checkequal(False, '', 'isalpha') self.checkequal(True, 'a', 'isalpha') self.checkequal(True, 'A', 'isalpha') self.checkequal(False, '\n', 'isalpha') self.checkequal(True, 'abc', 'isalpha') self.checkequal(False, 'aBc123', 'isalpha') self.checkequal(False, 'abc\n', 'isalpha') self.checkraises(TypeError, 'abc', 'isalpha', 42) def test_isalnum(self): self.checkequal(False, '', 'isalnum') self.checkequal(True, 'a', 'isalnum') self.checkequal(True, 'A', 'isalnum') self.checkequal(False, '\n', 'isalnum') self.checkequal(True, '123abc456', 'isalnum') self.checkequal(True, 'a1b3c', 'isalnum') self.checkequal(False, 'aBc000 ', 'isalnum') self.checkequal(False, 'abc\n', 'isalnum') self.checkraises(TypeError, 'abc', 'isalnum', 42) def test_isdigit(self): self.checkequal(False, '', 'isdigit') self.checkequal(False, 'a', 'isdigit') self.checkequal(True, '0', 'isdigit') self.checkequal(True, '0123456789', 'isdigit') self.checkequal(False, '0123456789a', 'isdigit') self.checkraises(TypeError, 'abc', 'isdigit', 42) def test_title(self): self.checkequal(' Hello ', ' hello ', 'title') self.checkequal('Hello ', 'hello ', 'title') self.checkequal('Hello ', 'Hello ', 'title') self.checkequal('Format This As Title String', "fOrMaT thIs aS titLe String", 'title') self.checkequal('Format,This-As*Title;String', "fOrMaT,thIs-aS*titLe;String", 'title', ) self.checkequal('Getint', "getInt", 'title') self.checkraises(TypeError, 'hello', 'title', 42) def test_splitlines(self): self.checkequal(['abc', 'def', '', 'ghi'], "abc\ndef\n\rghi", 'splitlines') self.checkequal(['abc', 'def', '', 'ghi'], "abc\ndef\n\r\nghi", 'splitlines') self.checkequal(['abc', 'def', 'ghi'], "abc\ndef\r\nghi", 'splitlines') self.checkequal(['abc', 'def', 'ghi'], "abc\ndef\r\nghi\n", 'splitlines') self.checkequal(['abc', 'def', 'ghi', ''], "abc\ndef\r\nghi\n\r", 'splitlines') self.checkequal(['', 'abc', 'def', 'ghi', ''], "\nabc\ndef\r\nghi\n\r", 'splitlines') self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'], "\nabc\ndef\r\nghi\n\r", 'splitlines', 1) self.checkraises(TypeError, 'abc', 'splitlines', 42, 42) class MixinStrUnicodeUserStringTest(NonStringModuleTest): # additional tests that only work for # stringlike objects, i.e. str, unicode, UserString # (but not the string module) def test_startswith(self): self.checkequal(True, 'hello', 'startswith', 'he') self.checkequal(True, 'hello', 'startswith', 'hello') self.checkequal(False, 'hello', 'startswith', 'hello world') self.checkequal(True, 'hello', 'startswith', '') self.checkequal(False, 'hello', 'startswith', 'ello') self.checkequal(True, 'hello', 'startswith', 'ello', 1) self.checkequal(True, 'hello', 'startswith', 'o', 4) self.checkequal(False, 'hello', 'startswith', 'o', 5) self.checkequal(True, 'hello', 'startswith', '', 5) self.checkequal(False, 'hello', 'startswith', 'lo', 6) self.checkequal(True, 'helloworld', 'startswith', 'lowo', 3) self.checkequal(True, 'helloworld', 'startswith', 'lowo', 3, 7) self.checkequal(False, 'helloworld', 'startswith', 'lowo', 3, 6) # test negative indices self.checkequal(True, 'hello', 'startswith', 'he', 0, -1) self.checkequal(True, 'hello', 'startswith', 'he', -53, -1) self.checkequal(False, 'hello', 'startswith', 'hello', 0, -1) self.checkequal(False, 'hello', 'startswith', 'hello world', -1, -10) self.checkequal(False, 'hello', 'startswith', 'ello', -5) self.checkequal(True, 'hello', 'startswith', 'ello', -4) self.checkequal(False, 'hello', 'startswith', 'o', -2) self.checkequal(True, 'hello', 'startswith', 'o', -1) self.checkequal(True, 'hello', 'startswith', '', -3, -3) self.checkequal(False, 'hello', 'startswith', 'lo', -9) self.checkraises(TypeError, 'hello', 'startswith') self.checkraises(TypeError, 'hello', 'startswith', 42) # test tuple arguments self.checkequal(True, 'hello', 'startswith', ('he', 'ha')) self.checkequal(False, 'hello', 'startswith', ('lo', 'llo')) self.checkequal(True, 'hello', 'startswith', ('hellox', 'hello')) self.checkequal(False, 'hello', 'startswith', ()) self.checkequal(True, 'helloworld', 'startswith', ('hellowo', 'rld', 'lowo'), 3) self.checkequal(False, 'helloworld', 'startswith', ('hellowo', 'ello', 'rld'), 3) self.checkequal(True, 'hello', 'startswith', ('lo', 'he'), 0, -1) self.checkequal(False, 'hello', 'startswith', ('he', 'hel'), 0, 1) self.checkequal(True, 'hello', 'startswith', ('he', 'hel'), 0, 2) self.checkraises(TypeError, 'hello', 'startswith', (42,)) def test_endswith(self): self.checkequal(True, 'hello', 'endswith', 'lo') self.checkequal(False, 'hello', 'endswith', 'he') self.checkequal(True, 'hello', 'endswith', '') self.checkequal(False, 'hello', 'endswith', 'hello world') self.checkequal(False, 'helloworld', 'endswith', 'worl') self.checkequal(True, 'helloworld', 'endswith', 'worl', 3, 9) self.checkequal(True, 'helloworld', 'endswith', 'world', 3, 12) self.checkequal(True, 'helloworld', 'endswith', 'lowo', 1, 7) self.checkequal(True, 'helloworld', 'endswith', 'lowo', 2, 7) self.checkequal(True, 'helloworld', 'endswith', 'lowo', 3, 7) self.checkequal(False, 'helloworld', 'endswith', 'lowo', 4, 7) self.checkequal(False, 'helloworld', 'endswith', 'lowo', 3, 8) self.checkequal(False, 'ab', 'endswith', 'ab', 0, 1) self.checkequal(False, 'ab', 'endswith', 'ab', 0, 0) # test negative indices self.checkequal(True, 'hello', 'endswith', 'lo', -2) self.checkequal(False, 'hello', 'endswith', 'he', -2) self.checkequal(True, 'hello', 'endswith', '', -3, -3) self.checkequal(False, 'hello', 'endswith', 'hello world', -10, -2) self.checkequal(False, 'helloworld', 'endswith', 'worl', -6) self.checkequal(True, 'helloworld', 'endswith', 'worl', -5, -1) self.checkequal(True, 'helloworld', 'endswith', 'worl', -5, 9) self.checkequal(True, 'helloworld', 'endswith', 'world', -7, 12) self.checkequal(True, 'helloworld', 'endswith', 'lowo', -99, -3) self.checkequal(True, 'helloworld', 'endswith', 'lowo', -8, -3) self.checkequal(True, 'helloworld', 'endswith', 'lowo', -7, -3) self.checkequal(False, 'helloworld', 'endswith', 'lowo', 3, -4) self.checkequal(False, 'helloworld', 'endswith', 'lowo', -8, -2) self.checkraises(TypeError, 'hello', 'endswith') self.checkraises(TypeError, 'hello', 'endswith', 42) # test tuple arguments self.checkequal(False, 'hello', 'endswith', ('he', 'ha')) self.checkequal(True, 'hello', 'endswith', ('lo', 'llo')) self.checkequal(True, 'hello', 'endswith', ('hellox', 'hello')) self.checkequal(False, 'hello', 'endswith', ()) self.checkequal(True, 'helloworld', 'endswith', ('hellowo', 'rld', 'lowo'), 3) self.checkequal(False, 'helloworld', 'endswith', ('hellowo', 'ello', 'rld'), 3, -1) self.checkequal(True, 'hello', 'endswith', ('hell', 'ell'), 0, -1) self.checkequal(False, 'hello', 'endswith', ('he', 'hel'), 0, 1) self.checkequal(True, 'hello', 'endswith', ('he', 'hell'), 0, 4) self.checkraises(TypeError, 'hello', 'endswith', (42,)) def test___contains__(self): self.checkequal(True, '', '__contains__', '') self.checkequal(True, 'abc', '__contains__', '') self.checkequal(False, 'abc', '__contains__', '\0') self.checkequal(True, '\0abc', '__contains__', '\0') self.checkequal(True, 'abc\0', '__contains__', '\0') self.checkequal(True, '\0abc', '__contains__', 'a') self.checkequal(True, 'asdf', '__contains__', 'asdf') self.checkequal(False, 'asd', '__contains__', 'asdf') self.checkequal(False, '', '__contains__', 'asdf') def test_subscript(self): self.checkequal(u'a', 'abc', '__getitem__', 0) self.checkequal(u'c', 'abc', '__getitem__', -1) self.checkequal(u'a', 'abc', '__getitem__', 0L) self.checkequal(u'abc', 'abc', '__getitem__', slice(0, 3)) self.checkequal(u'abc', 'abc', '__getitem__', slice(0, 1000)) self.checkequal(u'a', 'abc', '__getitem__', slice(0, 1)) self.checkequal(u'', 'abc', '__getitem__', slice(0, 0)) self.checkraises(TypeError, 'abc', '__getitem__', 'def') def test_slice(self): self.checkequal('abc', 'abc', '__getslice__', 0, 1000) self.checkequal('abc', 'abc', '__getslice__', 0, 3) self.checkequal('ab', 'abc', '__getslice__', 0, 2) self.checkequal('bc', 'abc', '__getslice__', 1, 3) self.checkequal('b', 'abc', '__getslice__', 1, 2) self.checkequal('', 'abc', '__getslice__', 2, 2) self.checkequal('', 'abc', '__getslice__', 1000, 1000) self.checkequal('', 'abc', '__getslice__', 2000, 1000) self.checkequal('', 'abc', '__getslice__', 2, 1) self.checkraises(TypeError, 'abc', '__getslice__', 'def') def test_extended_getslice(self): # Test extended slicing by comparing with list slicing. s = string.ascii_letters + string.digits indices = (0, None, 1, 3, 41, -1, -2, -37) for start in indices: for stop in indices: # Skip step 0 (invalid) for step in indices[1:]: L = list(s)[start:stop:step] self.checkequal(u"".join(L), s, '__getitem__', slice(start, stop, step)) def test_mul(self): self.checkequal('', 'abc', '__mul__', -1) self.checkequal('', 'abc', '__mul__', 0) self.checkequal('abc', 'abc', '__mul__', 1) self.checkequal('abcabcabc', 'abc', '__mul__', 3) self.checkraises(TypeError, 'abc', '__mul__') self.checkraises(TypeError, 'abc', '__mul__', '') # XXX: on a 64-bit system, this doesn't raise an overflow error, # but either raises a MemoryError, or succeeds (if you have 54TiB) #self.checkraises(OverflowError, 10000*'abc', '__mul__', 2000000000) def test_join(self): # join now works with any sequence type # moved here, because the argument order is # different in string.join (see the test in # test.test_string.StringTest.test_join) self.checkequal('a b c d', ' ', 'join', ['a', 'b', 'c', 'd']) self.checkequal('abcd', '', 'join', ('a', 'b', 'c', 'd')) self.checkequal('bd', '', 'join', ('', 'b', '', 'd')) self.checkequal('ac', '', 'join', ('a', '', 'c', '')) self.checkequal('w x y z', ' ', 'join', Sequence()) self.checkequal('abc', 'a', 'join', ('abc',)) self.checkequal('z', 'a', 'join', UserList(['z'])) if test_support.have_unicode: self.checkequal(unicode('a.b.c'), unicode('.'), 'join', ['a', 'b', 'c']) self.checkequal(unicode('a.b.c'), '.', 'join', [unicode('a'), 'b', 'c']) self.checkequal(unicode('a.b.c'), '.', 'join', ['a', unicode('b'), 'c']) self.checkequal(unicode('a.b.c'), '.', 'join', ['a', 'b', unicode('c')]) self.checkraises(TypeError, '.', 'join', ['a', unicode('b'), 3]) for i in [5, 25, 125]: self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join', ['a' * i] * i) self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join', ('a' * i,) * i) self.checkraises(TypeError, ' ', 'join', BadSeq1()) self.checkequal('a b c', ' ', 'join', BadSeq2()) self.checkraises(TypeError, ' ', 'join') self.checkraises(TypeError, ' ', 'join', None) self.checkraises(TypeError, ' ', 'join', 7) self.checkraises(TypeError, ' ', 'join', Sequence([7, 'hello', 123L])) try: def f(): yield 4 + "" self.fixtype(' ').join(f()) except TypeError, e: if '+' not in str(e): self.fail('join() ate exception message') else: self.fail('exception not raised') def test_formatting(self): self.checkequal('+hello+', '+%s+', '__mod__', 'hello') self.checkequal('+10+', '+%d+', '__mod__', 10) self.checkequal('a', "%c", '__mod__', "a") self.checkequal('a', "%c", '__mod__', "a") self.checkequal('"', "%c", '__mod__', 34) self.checkequal('$', "%c", '__mod__', 36) self.checkequal('10', "%d", '__mod__', 10) self.checkequal('\x7f', "%c", '__mod__', 0x7f) for ordinal in (-100, 0x200000): # unicode raises ValueError, str raises OverflowError self.checkraises((ValueError, OverflowError), '%c', '__mod__', ordinal) longvalue = sys.maxint + 10L slongvalue = str(longvalue) if slongvalue[-1] in ("L","l"): slongvalue = slongvalue[:-1] self.checkequal(' 42', '%3ld', '__mod__', 42) self.checkequal('42', '%d', '__mod__', 42L) self.checkequal('42', '%d', '__mod__', 42.0) self.checkequal(slongvalue, '%d', '__mod__', longvalue) self.checkcall('%d', '__mod__', float(longvalue)) self.checkequal('0042.00', '%07.2f', '__mod__', 42) self.checkequal('0042.00', '%07.2F', '__mod__', 42) self.checkraises(TypeError, 'abc', '__mod__') self.checkraises(TypeError, '%(foo)s', '__mod__', 42) self.checkraises(TypeError, '%s%s', '__mod__', (42,)) self.checkraises(TypeError, '%c', '__mod__', (None,)) self.checkraises(ValueError, '%(foo', '__mod__', {}) self.checkraises(TypeError, '%(foo)s %(bar)s', '__mod__', ('foo', 42)) self.checkraises(TypeError, '%d', '__mod__', "42") # not numeric # self.checkraises(TypeError, '%d', '__mod__', (42+0j)) # no int/long conversion provided # argument names with properly nested brackets are supported self.checkequal('bar', '%((foo))s', '__mod__', {'(foo)': 'bar'}) # 100 is a magic number in PyUnicode_Format, this forces a resize self.checkequal(103*'a'+'x', '%sx', '__mod__', 103*'a') self.checkraises(TypeError, '%*s', '__mod__', ('foo', 'bar')) self.checkraises(TypeError, '%10.*f', '__mod__', ('foo', 42.)) self.checkraises(ValueError, '%10', '__mod__', (42,)) class X(object): pass self.checkraises(TypeError, 'abc', '__mod__', X()) class X(Exception): def __getitem__(self, k): return k self.checkequal('melon apple', '%(melon)s %(apple)s', '__mod__', X()) # @test_support.cpython_only # def test_formatting_c_limits(self): # from _testcapi import PY_SSIZE_T_MAX, INT_MAX, UINT_MAX # SIZE_MAX = (1 << (PY_SSIZE_T_MAX.bit_length() + 1)) - 1 # width = int(PY_SSIZE_T_MAX + 1) # if width <= sys.maxint: # self.checkraises(OverflowError, '%*s', '__mod__', (width, '')) # prec = int(INT_MAX + 1) # if prec <= sys.maxint: # self.checkraises(OverflowError, '%.*f', '__mod__', (prec, 1. / 7)) # # Issue 15989 # width = int(SIZE_MAX + 1) # if width <= sys.maxint: # self.checkraises(OverflowError, '%*s', '__mod__', (width, '')) # prec = int(UINT_MAX + 1) # if prec <= sys.maxint: # self.checkraises(OverflowError, '%.*f', '__mod__', (prec, 1. / 7)) def test_floatformatting(self): # float formatting for prec in xrange(100): format = '%%.%if' % prec value = 0.01 for x in xrange(60): value = value * 3.14159265359 / 3.0 * 10.0 self.checkcall(format, "__mod__", value) def test_inplace_rewrites(self): # Check that strings don't copy and modify cached single-character strings self.checkequal('a', 'A', 'lower') self.checkequal(True, 'A', 'isupper') self.checkequal('A', 'a', 'upper') self.checkequal(True, 'a', 'islower') self.checkequal('a', 'A', 'replace', 'A', 'a') self.checkequal(True, 'A', 'isupper') self.checkequal('A', 'a', 'capitalize') self.checkequal(True, 'a', 'islower') self.checkequal('A', 'a', 'swapcase') self.checkequal(True, 'a', 'islower') self.checkequal('A', 'a', 'title') self.checkequal(True, 'a', 'islower') def test_partition(self): self.checkequal(('this is the par', 'ti', 'tion method'), 'this is the partition method', 'partition', 'ti') # from raymond's original specification S = 'http://www.python.org' self.checkequal(('http', '://', 'www.python.org'), S, 'partition', '://') self.checkequal(('http://www.python.org', '', ''), S, 'partition', '?') self.checkequal(('', 'http://', 'www.python.org'), S, 'partition', 'http://') self.checkequal(('http://www.python.', 'org', ''), S, 'partition', 'org') self.checkraises(ValueError, S, 'partition', '') self.checkraises(TypeError, S, 'partition', None) # mixed use of str and unicode self.assertEqual('a/b/c'.partition(u'/'), ('a', '/', 'b/c')) def test_rpartition(self): self.checkequal(('this is the rparti', 'ti', 'on method'), 'this is the rpartition method', 'rpartition', 'ti') # from raymond's original specification S = 'http://www.python.org' self.checkequal(('http', '://', 'www.python.org'), S, 'rpartition', '://') self.checkequal(('', '', 'http://www.python.org'), S, 'rpartition', '?') self.checkequal(('', 'http://', 'www.python.org'), S, 'rpartition', 'http://') self.checkequal(('http://www.python.', 'org', ''), S, 'rpartition', 'org') self.checkraises(ValueError, S, 'rpartition', '') self.checkraises(TypeError, S, 'rpartition', None) # mixed use of str and unicode self.assertEqual('a/b/c'.rpartition(u'/'), ('a/b', '/', 'c')) def test_none_arguments(self): # issue 11828 s = 'hello' self.checkequal(2, s, 'find', 'l', None) self.checkequal(3, s, 'find', 'l', -2, None) self.checkequal(2, s, 'find', 'l', None, -2) self.checkequal(0, s, 'find', 'h', None, None) self.checkequal(3, s, 'rfind', 'l', None) self.checkequal(3, s, 'rfind', 'l', -2, None) self.checkequal(2, s, 'rfind', 'l', None, -2) self.checkequal(0, s, 'rfind', 'h', None, None) self.checkequal(2, s, 'index', 'l', None) self.checkequal(3, s, 'index', 'l', -2, None) self.checkequal(2, s, 'index', 'l', None, -2) self.checkequal(0, s, 'index', 'h', None, None) self.checkequal(3, s, 'rindex', 'l', None) self.checkequal(3, s, 'rindex', 'l', -2, None) self.checkequal(2, s, 'rindex', 'l', None, -2) self.checkequal(0, s, 'rindex', 'h', None, None) self.checkequal(2, s, 'count', 'l', None) self.checkequal(1, s, 'count', 'l', -2, None) self.checkequal(1, s, 'count', 'l', None, -2) self.checkequal(0, s, 'count', 'x', None, None) self.checkequal(True, s, 'endswith', 'o', None) self.checkequal(True, s, 'endswith', 'lo', -2, None) self.checkequal(True, s, 'endswith', 'l', None, -2) self.checkequal(False, s, 'endswith', 'x', None, None) self.checkequal(True, s, 'startswith', 'h', None) self.checkequal(True, s, 'startswith', 'l', -2, None) self.checkequal(True, s, 'startswith', 'h', None, -2) self.checkequal(False, s, 'startswith', 'x', None, None) def test_find_etc_raise_correct_error_messages(self): # issue 11828 s = 'hello' x = 'x' self.assertRaisesRegexp(TypeError, r'\bfind\b', s.find, x, None, None, None) self.assertRaisesRegexp(TypeError, r'\brfind\b', s.rfind, x, None, None, None) self.assertRaisesRegexp(TypeError, r'\bindex\b', s.index, x, None, None, None) self.assertRaisesRegexp(TypeError, r'\brindex\b', s.rindex, x, None, None, None) self.assertRaisesRegexp(TypeError, r'^count\(', s.count, x, None, None, None) self.assertRaisesRegexp(TypeError, r'^startswith\(', s.startswith, x, None, None, None) self.assertRaisesRegexp(TypeError, r'^endswith\(', s.endswith, x, None, None, None) class MixinStrStringUserStringTest(object): # Additional tests for 8bit strings, i.e. str, UserString and # the string module def test_maketrans(self): self.assertEqual( ''.join(map(chr, xrange(256))).replace('abc', 'xyz'), string.maketrans('abc', 'xyz') ) self.assertRaises(ValueError, string.maketrans, 'abc', 'xyzw') # def test_translate(self): # table = string.maketrans('abc', 'xyz') # self.checkequal('xyzxyz', 'xyzabcdef', 'translate', table, 'def') # # table = string.maketrans('a', 'A') # self.checkequal('Abc', 'abc', 'translate', table) # self.checkequal('xyz', 'xyz', 'translate', table) # self.checkequal('yz', 'xyz', 'translate', table, 'x') # self.checkequal('yx', 'zyzzx', 'translate', None, 'z') # self.checkequal('zyzzx', 'zyzzx', 'translate', None, '') # self.checkequal('zyzzx', 'zyzzx', 'translate', None) # self.checkraises(ValueError, 'xyz', 'translate', 'too short', 'strip') # self.checkraises(ValueError, 'xyz', 'translate', 'too short') class MixinStrUserStringTest(object): # Additional tests that only work with # 8bit compatible object, i.e. str and UserString @unittest.skipUnless(test_support.have_unicode, 'no unicode support') def test_encoding_decoding(self): codecs = [('rot13', 'uryyb jbeyq'), ('base64', 'aGVsbG8gd29ybGQ=\n'), ('hex', '68656c6c6f20776f726c64'), ('uu', 'begin 666 \n+:&5L;&\\@=V]R;&0 \n \nend\n')] for encoding, data in codecs: with test_support.check_py3k_warnings(): self.checkequal(data, 'hello world', 'encode', encoding) with test_support.check_py3k_warnings(): self.checkequal('hello world', data, 'decode', encoding) # zlib is optional, so we make the test optional too... # try: # import zlib # except ImportError: # pass # else: # data = 'x\x9c\xcbH\xcd\xc9\xc9W(\xcf/\xcaI\x01\x00\x1a\x0b\x04]' # with test_support.check_py3k_warnings(): # self.checkequal(data, 'hello world', 'encode', 'zlib') # with test_support.check_py3k_warnings(): # self.checkequal('hello world', data, 'decode', 'zlib') self.checkraises(TypeError, 'xyz', 'decode', 42) self.checkraises(TypeError, 'xyz', 'encode', 42) class MixinStrUnicodeTest(object): # Additional tests that only work with str and unicode. def test_bug1001011(self): # Make sure join returns a NEW object for single item sequences # involving a subclass. # Make sure that it is of the appropriate type. # Check the optimisation still occurs for standard objects. t = self.type2test class subclass(t): pass s1 = subclass("abcd") s2 = t().join([s1]) self.assertTrue(s1 is not s2) self.assertTrue(type(s2) is t) s1 = t("abcd") s2 = t().join([s1]) self.assertTrue(s1 is s2) # Should also test mixed-type join. if t is unicode: s1 = subclass("abcd") s2 = "".join([s1]) self.assertTrue(s1 is not s2) self.assertTrue(type(s2) is t) s1 = t("abcd") s2 = "".join([s1]) self.assertTrue(s1 is s2) elif t is str: s1 = subclass("abcd") s2 = u"".join([s1]) self.assertTrue(s1 is not s2) self.assertTrue(type(s2) is unicode) # promotes! s1 = t("abcd") s2 = u"".join([s1]) self.assertTrue(s1 is not s2) self.assertTrue(type(s2) is unicode) # promotes! else: self.fail("unexpected type for MixinStrUnicodeTest %r" % t) ================================================ FILE: third_party/stdlib/test/test_argparse.py ================================================ # Author: Steven J. Bethard . # import codecs # import inspect import os # import shutil # import stat import sys # import tempfile import textwrap import unittest import argparse # from StringIO import StringIO import StringIO as _StringIO StringIO = _StringIO.StringIO class StdIOBuffer(StringIO): pass from test import test_support class TestCase(unittest.TestCase): def assertEqual(self, obj1, obj2): if obj1 != obj2: print('') print(repr(obj1)) print(repr(obj2)) print(obj1) print(obj2) super(TestCase, self).assertEqual(obj1, obj2) def setUp(self): # The tests assume that line wrapping occurs at 80 columns, but this # behaviour can be overridden by setting the COLUMNS environment # variable. To ensure that this assumption is true, unset COLUMNS. env = test_support.EnvironmentVarGuard() env.unset("COLUMNS") self.addCleanup(env.__exit__) # class TempDirMixin(object): # def setUp(self): # self.temp_dir = tempfile.mkdtemp() # self.old_dir = os.getcwd() # os.chdir(self.temp_dir) # def tearDown(self): # os.chdir(self.old_dir) # for root, dirs, files in os.walk(self.temp_dir, topdown=False): # for name in files: # # os.chmod(os.path.join(self.temp_dir, name), stat.S_IWRITE) # os.chmod(os.path.join(self.temp_dir, name), 0o200) # # TODO: Add shutil to stdlib. # # shutil.rmtree(self.temp_dir, True) # def create_readonly_file(self, filename): # file_path = os.path.join(self.temp_dir, filename) # with open(file_path, 'w') as file: # file.write(filename) # # os.chmod(file_path, stat.S_IREAD) # os.chmod(file_path, 0o400) class Sig(object): def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs class NS(object): def __init__(self, **kwargs): self.__dict__.update(kwargs) def __repr__(self): sorted_items = sorted(self.__dict__.items()) kwarg_str = ', '.join(['%s=%r' % tup for tup in sorted_items]) return '%s(%s)' % (type(self).__name__, kwarg_str) __hash__ = None def __eq__(self, other): # return vars(self) == vars(other) return self.__dict__ == other.__dict__ def __ne__(self, other): return not (self == other) class ArgumentParserError(Exception): def __init__(self, message, stdout=None, stderr=None, error_code=None): Exception.__init__(self, message, stdout, stderr) self.message = message self.stdout = stdout self.stderr = stderr self.error_code = error_code def stderr_to_parser_error(parse_args, *args, **kwargs): # if this is being called recursively and stderr or stdout is already being # redirected, simply call the function and let the enclosing function # catch the exception if isinstance(sys.stderr, StdIOBuffer) or isinstance(sys.stdout, StdIOBuffer): # return parse_args(*args, **kwargs) try: return parse_args(*args, **kwargs) except SystemExit: stdout = stderr = None if isinstance(sys.stdout, StdIOBuffer): stdout = sys.stdout.getvalue() sys.stdout.truncate(0) if isinstance(sys.stderr, StdIOBuffer): stderr = sys.stderr.getvalue() sys.stderr.truncate(0) raise ArgumentParserError("SystemExit", stdout, stderr) # if this is not being called recursively, redirect stderr and # use it as the ArgumentParserError message old_stdout = sys.stdout old_stderr = sys.stderr sys.stdout = StdIOBuffer() sys.stderr = StdIOBuffer() try: try: result = parse_args(*args, **kwargs) # for key in list(vars(result)): for key in result.__dict__: if getattr(result, key) is sys.stdout: setattr(result, key, old_stdout) if getattr(result, key) is sys.stderr: setattr(result, key, old_stderr) return result except SystemExit: code = sys.exc_info()[1].code stdout = sys.stdout.getvalue() stderr = sys.stderr.getvalue() raise ArgumentParserError("SystemExit", stdout, stderr, code) finally: sys.stdout = old_stdout sys.stderr = old_stderr class ErrorRaisingArgumentParser(argparse.ArgumentParser): def parse_args(self, *args, **kwargs): parse_args = super(ErrorRaisingArgumentParser, self).parse_args return stderr_to_parser_error(parse_args, *args, **kwargs) def exit(self, *args, **kwargs): exit = super(ErrorRaisingArgumentParser, self).exit return stderr_to_parser_error(exit, *args, **kwargs) def error(self, *args, **kwargs): error = super(ErrorRaisingArgumentParser, self).error return stderr_to_parser_error(error, *args, **kwargs) class ParserTesterMetaclass(type): """Adds parser tests using the class attributes. Classes of this type should specify the following attributes: argument_signatures -- a list of Sig objects which specify the signatures of Argument objects to be created failures -- a list of args lists that should cause the parser to fail successes -- a list of (initial_args, options, remaining_args) tuples where initial_args specifies the string args to be parsed, options is a dict that should match the vars() of the options parsed out of initial_args, and remaining_args should be any remaining unparsed arguments """ def __init__(cls, name, bases, bodydict): if name == 'ParserTestCase': return # default parser signature is empty if not hasattr(cls, 'parser_signature'): cls.parser_signature = Sig() if not hasattr(cls, 'parser_class'): cls.parser_class = ErrorRaisingArgumentParser # --------------------------------------- # functions for adding optional arguments # --------------------------------------- def no_groups(parser, argument_signatures): """Add all arguments directly to the parser""" for sig in argument_signatures: parser.add_argument(*sig.args, **sig.kwargs) def one_group(parser, argument_signatures): """Add all arguments under a single group in the parser""" group = parser.add_argument_group('foo') for sig in argument_signatures: group.add_argument(*sig.args, **sig.kwargs) def many_groups(parser, argument_signatures): """Add each argument in its own group to the parser""" for i, sig in enumerate(argument_signatures): # group = parser.add_argument_group('foo:%i' % i) group = parser.add_argument_group('foo:%d' % i) group.add_argument(*sig.args, **sig.kwargs) # -------------------------- # functions for parsing args # -------------------------- def listargs(parser, args): """Parse the args by passing in a list""" return parser.parse_args(args) def sysargs(parser, args): """Parse the args by defaulting to sys.argv""" old_sys_argv = sys.argv sys.argv = [old_sys_argv[0]] + args try: return parser.parse_args() finally: sys.argv = old_sys_argv # class that holds the combination of one optional argument # addition method and one arg parsing method class AddTests(object): def __init__(self, tester_cls, add_arguments, parse_args): self._add_arguments = add_arguments self._parse_args = parse_args add_arguments_name = self._add_arguments.__name__ parse_args_name = self._parse_args.__name__ for test_func in [self.test_failures, self.test_successes]: func_name = test_func.__name__ names = func_name, add_arguments_name, parse_args_name test_name = '_'.join(names) def wrapper(self, test_func=test_func): test_func(self) # try: # wrapper.__name__ = test_name # except TypeError: # pass setattr(tester_cls, test_name, wrapper) def _get_parser(self, tester): args = tester.parser_signature.args kwargs = tester.parser_signature.kwargs parser = tester.parser_class(*args, **kwargs) self._add_arguments(parser, tester.argument_signatures) return parser def test_failures(self, tester): parser = self._get_parser(tester) for args_str in tester.failures: args = args_str.split() raises = tester.assertRaises raises(ArgumentParserError, parser.parse_args, args) def test_successes(self, tester): parser = self._get_parser(tester) for args, expected_ns in tester.successes: if isinstance(args, str): args = args.split() result_ns = self._parse_args(parser, args) tester.assertEqual(expected_ns, result_ns) # add tests for each combination of an optionals adding method # and an arg parsing method for add_arguments in [no_groups, one_group, many_groups]: for parse_args in [listargs, sysargs]: AddTests(cls, add_arguments, parse_args) bases = TestCase, ParserTestCase = ParserTesterMetaclass('ParserTestCase', bases, {}) # =============== # Optionals tests # =============== class TestOptionalsSingleDash(ParserTestCase): """Test an Optional with a single-dash option string""" argument_signatures = [Sig('-x')] failures = ['-x', 'a', '--foo', '-x --foo', '-x -y'] successes = [ ('', NS(x=None)), ('-x a', NS(x='a')), ('-xa', NS(x='a')), ('-x -1', NS(x='-1')), ('-x-1', NS(x='-1')), ] class TestOptionalsSingleDashCombined(ParserTestCase): """Test an Optional with a single-dash option string""" argument_signatures = [ Sig('-x', action='store_true'), Sig('-yyy', action='store_const', const=42), Sig('-z'), ] failures = ['a', '--foo', '-xa', '-x --foo', '-x -z', '-z -x', '-yx', '-yz a', '-yyyx', '-yyyza', '-xyza'] successes = [ ('', NS(x=False, yyy=None, z=None)), ('-x', NS(x=True, yyy=None, z=None)), ('-za', NS(x=False, yyy=None, z='a')), ('-z a', NS(x=False, yyy=None, z='a')), ('-xza', NS(x=True, yyy=None, z='a')), ('-xz a', NS(x=True, yyy=None, z='a')), ('-x -za', NS(x=True, yyy=None, z='a')), ('-x -z a', NS(x=True, yyy=None, z='a')), ('-y', NS(x=False, yyy=42, z=None)), ('-yyy', NS(x=False, yyy=42, z=None)), ('-x -yyy -za', NS(x=True, yyy=42, z='a')), ('-x -yyy -z a', NS(x=True, yyy=42, z='a')), ] class TestOptionalsSingleDashLong(ParserTestCase): """Test an Optional with a multi-character single-dash option string""" argument_signatures = [Sig('-foo')] failures = ['-foo', 'a', '--foo', '-foo --foo', '-foo -y', '-fooa'] successes = [ ('', NS(foo=None)), ('-foo a', NS(foo='a')), ('-foo -1', NS(foo='-1')), ('-fo a', NS(foo='a')), ('-f a', NS(foo='a')), ] class TestOptionalsSingleDashSubsetAmbiguous(ParserTestCase): """Test Optionals where option strings are subsets of each other""" argument_signatures = [Sig('-f'), Sig('-foobar'), Sig('-foorab')] failures = ['-f', '-foo', '-fo', '-foo b', '-foob', '-fooba', '-foora'] successes = [ ('', NS(f=None, foobar=None, foorab=None)), ('-f a', NS(f='a', foobar=None, foorab=None)), ('-fa', NS(f='a', foobar=None, foorab=None)), ('-foa', NS(f='oa', foobar=None, foorab=None)), ('-fooa', NS(f='ooa', foobar=None, foorab=None)), ('-foobar a', NS(f=None, foobar='a', foorab=None)), ('-foorab a', NS(f=None, foobar=None, foorab='a')), ] class TestOptionalsSingleDashAmbiguous(ParserTestCase): """Test Optionals that partially match but are not subsets""" argument_signatures = [Sig('-foobar'), Sig('-foorab')] failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b'] successes = [ ('', NS(foobar=None, foorab=None)), ('-foob a', NS(foobar='a', foorab=None)), ('-foor a', NS(foobar=None, foorab='a')), ('-fooba a', NS(foobar='a', foorab=None)), ('-foora a', NS(foobar=None, foorab='a')), ('-foobar a', NS(foobar='a', foorab=None)), ('-foorab a', NS(foobar=None, foorab='a')), ] class TestOptionalsNumeric(ParserTestCase): """Test an Optional with a short opt string""" argument_signatures = [Sig('-1', dest='one')] failures = ['-1', 'a', '-1 --foo', '-1 -y', '-1 -1', '-1 -2'] successes = [ ('', NS(one=None)), ('-1 a', NS(one='a')), ('-1a', NS(one='a')), ('-1-2', NS(one='-2')), ] class TestOptionalsDoubleDash(ParserTestCase): """Test an Optional with a double-dash option string""" argument_signatures = [Sig('--foo')] failures = ['--foo', '-f', '-f a', 'a', '--foo -x', '--foo --bar'] successes = [ ('', NS(foo=None)), ('--foo a', NS(foo='a')), ('--foo=a', NS(foo='a')), ('--foo -2.5', NS(foo='-2.5')), ('--foo=-2.5', NS(foo='-2.5')), ] class TestOptionalsDoubleDashPartialMatch(ParserTestCase): """Tests partial matching with a double-dash option string""" argument_signatures = [ Sig('--badger', action='store_true'), Sig('--bat'), ] failures = ['--bar', '--b', '--ba', '--b=2', '--ba=4', '--badge 5'] successes = [ ('', NS(badger=False, bat=None)), ('--bat X', NS(badger=False, bat='X')), ('--bad', NS(badger=True, bat=None)), ('--badg', NS(badger=True, bat=None)), ('--badge', NS(badger=True, bat=None)), ('--badger', NS(badger=True, bat=None)), ] class TestOptionalsDoubleDashPrefixMatch(ParserTestCase): """Tests when one double-dash option string is a prefix of another""" argument_signatures = [ Sig('--badger', action='store_true'), Sig('--ba'), ] failures = ['--bar', '--b', '--ba', '--b=2', '--badge 5'] successes = [ ('', NS(badger=False, ba=None)), ('--ba X', NS(badger=False, ba='X')), ('--ba=X', NS(badger=False, ba='X')), ('--bad', NS(badger=True, ba=None)), ('--badg', NS(badger=True, ba=None)), ('--badge', NS(badger=True, ba=None)), ('--badger', NS(badger=True, ba=None)), ] class TestOptionalsSingleDoubleDash(ParserTestCase): """Test an Optional with single- and double-dash option strings""" argument_signatures = [ Sig('-f', action='store_true'), Sig('--bar'), Sig('-baz', action='store_const', const=42), ] failures = ['--bar', '-fbar', '-fbaz', '-bazf', '-b B', 'B'] successes = [ ('', NS(f=False, bar=None, baz=None)), ('-f', NS(f=True, bar=None, baz=None)), ('--ba B', NS(f=False, bar='B', baz=None)), ('-f --bar B', NS(f=True, bar='B', baz=None)), ('-f -b', NS(f=True, bar=None, baz=42)), ('-ba -f', NS(f=True, bar=None, baz=42)), ] class TestOptionalsAlternatePrefixChars(ParserTestCase): """Test an Optional with option strings with custom prefixes""" parser_signature = Sig(prefix_chars='+:/', add_help=False) argument_signatures = [ Sig('+f', action='store_true'), Sig('::bar'), Sig('/baz', action='store_const', const=42), ] failures = ['--bar', '-fbar', '-b B', 'B', '-f', '--bar B', '-baz', '-h', '--help', '+h', '::help', '/help'] successes = [ ('', NS(f=False, bar=None, baz=None)), ('+f', NS(f=True, bar=None, baz=None)), ('::ba B', NS(f=False, bar='B', baz=None)), ('+f ::bar B', NS(f=True, bar='B', baz=None)), ('+f /b', NS(f=True, bar=None, baz=42)), ('/ba +f', NS(f=True, bar=None, baz=42)), ] class TestOptionalsAlternatePrefixCharsAddedHelp(ParserTestCase): """When ``-`` not in prefix_chars, default operators created for help should use the prefix_chars in use rather than - or -- http://bugs.python.org/issue9444""" parser_signature = Sig(prefix_chars='+:/', add_help=True) argument_signatures = [ Sig('+f', action='store_true'), Sig('::bar'), Sig('/baz', action='store_const', const=42), ] failures = ['--bar', '-fbar', '-b B', 'B', '-f', '--bar B', '-baz'] successes = [ ('', NS(f=False, bar=None, baz=None)), ('+f', NS(f=True, bar=None, baz=None)), ('::ba B', NS(f=False, bar='B', baz=None)), ('+f ::bar B', NS(f=True, bar='B', baz=None)), ('+f /b', NS(f=True, bar=None, baz=42)), ('/ba +f', NS(f=True, bar=None, baz=42)) ] class TestOptionalsAlternatePrefixCharsMultipleShortArgs(ParserTestCase): """Verify that Optionals must be called with their defined prefixes""" parser_signature = Sig(prefix_chars='+-', add_help=False) argument_signatures = [ Sig('-x', action='store_true'), Sig('+y', action='store_true'), Sig('+z', action='store_true'), ] failures = ['-w', '-xyz', '+x', '-y', '+xyz', ] successes = [ ('', NS(x=False, y=False, z=False)), ('-x', NS(x=True, y=False, z=False)), ('+y -x', NS(x=True, y=True, z=False)), ('+yz -x', NS(x=True, y=True, z=True)), ] class TestOptionalsShortLong(ParserTestCase): """Test a combination of single- and double-dash option strings""" argument_signatures = [ Sig('-v', '--verbose', '-n', '--noisy', action='store_true'), ] failures = ['--x --verbose', '-N', 'a', '-v x'] successes = [ ('', NS(verbose=False)), ('-v', NS(verbose=True)), ('--verbose', NS(verbose=True)), ('-n', NS(verbose=True)), ('--noisy', NS(verbose=True)), ] class TestOptionalsDest(ParserTestCase): """Tests various means of setting destination""" argument_signatures = [Sig('--foo-bar'), Sig('--baz', dest='zabbaz')] failures = ['a'] successes = [ ('--foo-bar f', NS(foo_bar='f', zabbaz=None)), ('--baz g', NS(foo_bar=None, zabbaz='g')), ('--foo-bar h --baz i', NS(foo_bar='h', zabbaz='i')), ('--baz j --foo-bar k', NS(foo_bar='k', zabbaz='j')), ] class TestOptionalsDefault(ParserTestCase): """Tests specifying a default for an Optional""" argument_signatures = [Sig('-x'), Sig('-y', default=42)] failures = ['a'] successes = [ ('', NS(x=None, y=42)), ('-xx', NS(x='x', y=42)), ('-yy', NS(x=None, y='y')), ] class TestOptionalsNargsDefault(ParserTestCase): """Tests not specifying the number of args for an Optional""" argument_signatures = [Sig('-x')] failures = ['a', '-x'] successes = [ ('', NS(x=None)), ('-x a', NS(x='a')), ] class TestOptionalsNargs1(ParserTestCase): """Tests specifying 1 arg for an Optional""" argument_signatures = [Sig('-x', nargs=1)] failures = ['a', '-x'] successes = [ ('', NS(x=None)), ('-x a', NS(x=['a'])), ] class TestOptionalsNargs3(ParserTestCase): """Tests specifying 3 args for an Optional""" argument_signatures = [Sig('-x', nargs=3)] failures = ['a', '-x', '-x a', '-x a b', 'a -x', 'a -x b'] successes = [ ('', NS(x=None)), ('-x a b c', NS(x=['a', 'b', 'c'])), ] class TestOptionalsNargsOptional(ParserTestCase): """Tests specifying an Optional arg for an Optional""" argument_signatures = [ Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), Sig('-z', nargs='?', type=int, const='42', default='84'), ] failures = ['2'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), ('-w 2', NS(w='2', x=None, y='spam', z=84)), ('-x', NS(w=None, x=42, y='spam', z=84)), ('-x 2', NS(w=None, x='2', y='spam', z=84)), ('-y', NS(w=None, x=None, y=None, z=84)), ('-y 2', NS(w=None, x=None, y='2', z=84)), ('-z', NS(w=None, x=None, y='spam', z=42)), ('-z 2', NS(w=None, x=None, y='spam', z=2)), ] class TestOptionalsNargsZeroOrMore(ParserTestCase): """Tests specifying args for an Optional that accepts zero or more""" argument_signatures = [ Sig('-x', nargs='*'), Sig('-y', nargs='*', default='spam'), ] failures = ['a'] successes = [ ('', NS(x=None, y='spam')), ('-x', NS(x=[], y='spam')), ('-x a', NS(x=['a'], y='spam')), ('-x a b', NS(x=['a', 'b'], y='spam')), ('-y', NS(x=None, y=[])), ('-y a', NS(x=None, y=['a'])), ('-y a b', NS(x=None, y=['a', 'b'])), ] class TestOptionalsNargsOneOrMore(ParserTestCase): """Tests specifying args for an Optional that accepts one or more""" argument_signatures = [ Sig('-x', nargs='+'), Sig('-y', nargs='+', default='spam'), ] failures = ['a', '-x', '-y', 'a -x', 'a -y b'] successes = [ ('', NS(x=None, y='spam')), ('-x a', NS(x=['a'], y='spam')), ('-x a b', NS(x=['a', 'b'], y='spam')), ('-y a', NS(x=None, y=['a'])), ('-y a b', NS(x=None, y=['a', 'b'])), ] class TestOptionalsChoices(ParserTestCase): """Tests specifying the choices for an Optional""" argument_signatures = [ Sig('-f', choices='abc'), Sig('-g', type=int, choices=range(5))] failures = ['a', '-f d', '-fad', '-ga', '-g 6'] successes = [ ('', NS(f=None, g=None)), ('-f a', NS(f='a', g=None)), ('-f c', NS(f='c', g=None)), ('-g 0', NS(f=None, g=0)), ('-g 03', NS(f=None, g=3)), ('-fb -g4', NS(f='b', g=4)), ] class TestOptionalsRequired(ParserTestCase): """Tests an optional action that is required""" argument_signatures = [ Sig('-x', type=int, required=True), ] failures = ['a', ''] successes = [ ('-x 1', NS(x=1)), ('-x42', NS(x=42)), ] class TestOptionalsActionStore(ParserTestCase): """Tests the store action for an Optional""" argument_signatures = [Sig('-x', action='store')] failures = ['a', 'a -x'] successes = [ ('', NS(x=None)), ('-xfoo', NS(x='foo')), ] class TestOptionalsActionStoreConst(ParserTestCase): """Tests the store_const action for an Optional""" argument_signatures = [Sig('-y', action='store_const', const=object)] failures = ['a'] successes = [ ('', NS(y=None)), ('-y', NS(y=object)), ] class TestOptionalsActionStoreFalse(ParserTestCase): """Tests the store_false action for an Optional""" argument_signatures = [Sig('-z', action='store_false')] failures = ['a', '-za', '-z a'] successes = [ ('', NS(z=True)), ('-z', NS(z=False)), ] class TestOptionalsActionStoreTrue(ParserTestCase): """Tests the store_true action for an Optional""" argument_signatures = [Sig('--apple', action='store_true')] failures = ['a', '--apple=b', '--apple b'] successes = [ ('', NS(apple=False)), ('--apple', NS(apple=True)), ] class TestOptionalsActionAppend(ParserTestCase): """Tests the append action for an Optional""" argument_signatures = [Sig('--baz', action='append')] failures = ['a', '--baz', 'a --baz', '--baz a b'] successes = [ ('', NS(baz=None)), ('--baz a', NS(baz=['a'])), ('--baz a --baz b', NS(baz=['a', 'b'])), ] class TestOptionalsActionAppendWithDefault(ParserTestCase): """Tests the append action for an Optional""" argument_signatures = [Sig('--baz', action='append', default=['X'])] failures = ['a', '--baz', 'a --baz', '--baz a b'] successes = [ ('', NS(baz=['X'])), ('--baz a', NS(baz=['X', 'a'])), ('--baz a --baz b', NS(baz=['X', 'a', 'b'])), ] class TestOptionalsActionAppendConst(ParserTestCase): """Tests the append_const action for an Optional""" argument_signatures = [ Sig('-b', action='append_const', const=Exception), Sig('-c', action='append', dest='b'), ] failures = ['a', '-c', 'a -c', '-bx', '-b x'] successes = [ ('', NS(b=None)), ('-b', NS(b=[Exception])), ('-b -cx -b -cyz', NS(b=[Exception, 'x', Exception, 'yz'])), ] class TestOptionalsActionAppendConstWithDefault(ParserTestCase): """Tests the append_const action for an Optional""" argument_signatures = [ Sig('-b', action='append_const', const=Exception, default=['X']), Sig('-c', action='append', dest='b'), ] failures = ['a', '-c', 'a -c', '-bx', '-b x'] successes = [ ('', NS(b=['X'])), ('-b', NS(b=['X', Exception])), ('-b -cx -b -cyz', NS(b=['X', Exception, 'x', Exception, 'yz'])), ] class TestOptionalsActionCount(ParserTestCase): """Tests the count action for an Optional""" argument_signatures = [Sig('-x', action='count')] failures = ['a', '-x a', '-x b', '-x a -x b'] successes = [ ('', NS(x=None)), ('-x', NS(x=1)), ] # ================ # Positional tests # ================ class TestPositionalsNargsNone(ParserTestCase): """Test a Positional that doesn't specify nargs""" argument_signatures = [Sig('foo')] failures = ['', '-x', 'a b'] successes = [ ('a', NS(foo='a')), ] class TestPositionalsNargs1(ParserTestCase): """Test a Positional that specifies an nargs of 1""" argument_signatures = [Sig('foo', nargs=1)] failures = ['', '-x', 'a b'] successes = [ ('a', NS(foo=['a'])), ] class TestPositionalsNargs2(ParserTestCase): """Test a Positional that specifies an nargs of 2""" argument_signatures = [Sig('foo', nargs=2)] failures = ['', 'a', '-x', 'a b c'] successes = [ ('a b', NS(foo=['a', 'b'])), ] class TestPositionalsNargsZeroOrMore(ParserTestCase): """Test a Positional that specifies unlimited nargs""" argument_signatures = [Sig('foo', nargs='*')] failures = ['-x'] successes = [ ('', NS(foo=[])), ('a', NS(foo=['a'])), ('a b', NS(foo=['a', 'b'])), ] class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" argument_signatures = [Sig('foo', nargs='*', default='bar')] failures = ['-x'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), ('a b', NS(foo=['a', 'b'])), ] class TestPositionalsNargsOneOrMore(ParserTestCase): """Test a Positional that specifies one or more nargs""" argument_signatures = [Sig('foo', nargs='+')] failures = ['', '-x'] successes = [ ('a', NS(foo=['a'])), ('a b', NS(foo=['a', 'b'])), ] class TestPositionalsNargsOptional(ParserTestCase): """Tests an Optional Positional""" argument_signatures = [Sig('foo', nargs='?')] failures = ['-x', 'a b'] successes = [ ('', NS(foo=None)), ('a', NS(foo='a')), ] class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" argument_signatures = [Sig('foo', nargs='?', default=42)] failures = ['-x', 'a b'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), ] class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """Tests an Optional Positional with a default value that needs to be converted to the appropriate type. """ argument_signatures = [ Sig('foo', nargs='?', type=int, default='42'), ] failures = ['-x', 'a b', '1 2'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), ] class TestPositionalsNargsNoneNone(ParserTestCase): """Test two Positionals that don't specify nargs""" argument_signatures = [Sig('foo'), Sig('bar')] failures = ['', '-x', 'a', 'a b c'] successes = [ ('a b', NS(foo='a', bar='b')), ] class TestPositionalsNargsNone1(ParserTestCase): """Test a Positional with no nargs followed by one with 1""" argument_signatures = [Sig('foo'), Sig('bar', nargs=1)] failures = ['', '--foo', 'a', 'a b c'] successes = [ ('a b', NS(foo='a', bar=['b'])), ] class TestPositionalsNargs2None(ParserTestCase): """Test a Positional with 2 nargs followed by one with none""" argument_signatures = [Sig('foo', nargs=2), Sig('bar')] failures = ['', '--foo', 'a', 'a b', 'a b c d'] successes = [ ('a b c', NS(foo=['a', 'b'], bar='c')), ] class TestPositionalsNargsNoneZeroOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with unlimited""" argument_signatures = [Sig('foo'), Sig('bar', nargs='*')] failures = ['', '--foo'] successes = [ ('a', NS(foo='a', bar=[])), ('a b', NS(foo='a', bar=['b'])), ('a b c', NS(foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOneOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with one or more""" argument_signatures = [Sig('foo'), Sig('bar', nargs='+')] failures = ['', '--foo', 'a'] successes = [ ('a b', NS(foo='a', bar=['b'])), ('a b c', NS(foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOptional(ParserTestCase): """Test a Positional with no nargs followed by one with an Optional""" argument_signatures = [Sig('foo'), Sig('bar', nargs='?')] failures = ['', '--foo', 'a b c'] successes = [ ('a', NS(foo='a', bar=None)), ('a b', NS(foo='a', bar='b')), ] class TestPositionalsNargsZeroOrMoreNone(ParserTestCase): """Test a Positional with unlimited nargs followed by one with none""" argument_signatures = [Sig('foo', nargs='*'), Sig('bar')] failures = ['', '--foo'] successes = [ ('a', NS(foo=[], bar='a')), ('a b', NS(foo=['a'], bar='b')), ('a b c', NS(foo=['a', 'b'], bar='c')), ] class TestPositionalsNargsOneOrMoreNone(ParserTestCase): """Test a Positional with one or more nargs followed by one with none""" argument_signatures = [Sig('foo', nargs='+'), Sig('bar')] failures = ['', '--foo', 'a'] successes = [ ('a b', NS(foo=['a'], bar='b')), ('a b c', NS(foo=['a', 'b'], bar='c')), ] class TestPositionalsNargsOptionalNone(ParserTestCase): """Test a Positional with an Optional nargs followed by one with none""" argument_signatures = [Sig('foo', nargs='?', default=42), Sig('bar')] failures = ['', '--foo', 'a b c'] successes = [ ('a', NS(foo=42, bar='a')), ('a b', NS(foo='a', bar='b')), ] class TestPositionalsNargs2ZeroOrMore(ParserTestCase): """Test a Positional with 2 nargs followed by one with unlimited""" argument_signatures = [Sig('foo', nargs=2), Sig('bar', nargs='*')] failures = ['', '--foo', 'a'] successes = [ ('a b', NS(foo=['a', 'b'], bar=[])), ('a b c', NS(foo=['a', 'b'], bar=['c'])), ] class TestPositionalsNargs2OneOrMore(ParserTestCase): """Test a Positional with 2 nargs followed by one with one or more""" argument_signatures = [Sig('foo', nargs=2), Sig('bar', nargs='+')] failures = ['', '--foo', 'a', 'a b'] successes = [ ('a b c', NS(foo=['a', 'b'], bar=['c'])), ] class TestPositionalsNargs2Optional(ParserTestCase): """Test a Positional with 2 nargs followed by one optional""" argument_signatures = [Sig('foo', nargs=2), Sig('bar', nargs='?')] failures = ['', '--foo', 'a', 'a b c d'] successes = [ ('a b', NS(foo=['a', 'b'], bar=None)), ('a b c', NS(foo=['a', 'b'], bar='c')), ] class TestPositionalsNargsZeroOrMore1(ParserTestCase): """Test a Positional with unlimited nargs followed by one with 1""" argument_signatures = [Sig('foo', nargs='*'), Sig('bar', nargs=1)] failures = ['', '--foo', ] successes = [ ('a', NS(foo=[], bar=['a'])), ('a b', NS(foo=['a'], bar=['b'])), ('a b c', NS(foo=['a', 'b'], bar=['c'])), ] class TestPositionalsNargsOneOrMore1(ParserTestCase): """Test a Positional with one or more nargs followed by one with 1""" argument_signatures = [Sig('foo', nargs='+'), Sig('bar', nargs=1)] failures = ['', '--foo', 'a'] successes = [ ('a b', NS(foo=['a'], bar=['b'])), ('a b c', NS(foo=['a', 'b'], bar=['c'])), ] class TestPositionalsNargsOptional1(ParserTestCase): """Test a Positional with an Optional nargs followed by one with 1""" argument_signatures = [Sig('foo', nargs='?'), Sig('bar', nargs=1)] failures = ['', '--foo', 'a b c'] successes = [ ('a', NS(foo=None, bar=['a'])), ('a b', NS(foo='a', bar=['b'])), ] class TestPositionalsNargsNoneZeroOrMore1(ParserTestCase): """Test three Positionals: no nargs, unlimited nargs and 1 nargs""" argument_signatures = [ Sig('foo'), Sig('bar', nargs='*'), Sig('baz', nargs=1), ] failures = ['', '--foo', 'a'] successes = [ ('a b', NS(foo='a', bar=[], baz=['b'])), ('a b c', NS(foo='a', bar=['b'], baz=['c'])), ] class TestPositionalsNargsNoneOneOrMore1(ParserTestCase): """Test three Positionals: no nargs, one or more nargs and 1 nargs""" argument_signatures = [ Sig('foo'), Sig('bar', nargs='+'), Sig('baz', nargs=1), ] failures = ['', '--foo', 'a', 'b'] successes = [ ('a b c', NS(foo='a', bar=['b'], baz=['c'])), ('a b c d', NS(foo='a', bar=['b', 'c'], baz=['d'])), ] class TestPositionalsNargsNoneOptional1(ParserTestCase): """Test three Positionals: no nargs, optional narg and 1 nargs""" argument_signatures = [ Sig('foo'), Sig('bar', nargs='?', default=0.625), Sig('baz', nargs=1), ] failures = ['', '--foo', 'a'] successes = [ ('a b', NS(foo='a', bar=0.625, baz=['b'])), ('a b c', NS(foo='a', bar='b', baz=['c'])), ] class TestPositionalsNargsOptionalOptional(ParserTestCase): """Test two optional nargs""" argument_signatures = [ Sig('foo', nargs='?'), Sig('bar', nargs='?', default=42), ] failures = ['--foo', 'a b c'] successes = [ ('', NS(foo=None, bar=42)), ('a', NS(foo='a', bar=42)), ('a b', NS(foo='a', bar='b')), ] class TestPositionalsNargsOptionalZeroOrMore(ParserTestCase): """Test an Optional narg followed by unlimited nargs""" argument_signatures = [Sig('foo', nargs='?'), Sig('bar', nargs='*')] failures = ['--foo'] successes = [ ('', NS(foo=None, bar=[])), ('a', NS(foo='a', bar=[])), ('a b', NS(foo='a', bar=['b'])), ('a b c', NS(foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsOptionalOneOrMore(ParserTestCase): """Test an Optional narg followed by one or more nargs""" argument_signatures = [Sig('foo', nargs='?'), Sig('bar', nargs='+')] failures = ['', '--foo'] successes = [ ('a', NS(foo=None, bar=['a'])), ('a b', NS(foo='a', bar=['b'])), ('a b c', NS(foo='a', bar=['b', 'c'])), ] class TestPositionalsChoicesString(ParserTestCase): """Test a set of single-character choices""" argument_signatures = [Sig('spam', choices=set('abcdefg'))] failures = ['', '--foo', 'h', '42', 'ef'] successes = [ ('a', NS(spam='a')), ('g', NS(spam='g')), ] class TestPositionalsChoicesInt(ParserTestCase): """Test a set of integer choices""" argument_signatures = [Sig('spam', type=int, choices=range(20))] failures = ['', '--foo', 'h', '42', 'ef'] successes = [ ('4', NS(spam=4)), ('15', NS(spam=15)), ] class TestPositionalsActionAppend(ParserTestCase): """Test the 'append' action""" argument_signatures = [ Sig('spam', action='append'), Sig('spam', action='append', nargs=2), ] failures = ['', '--foo', 'a', 'a b', 'a b c d'] successes = [ ('a b c', NS(spam=['a', ['b', 'c']])), ] # ======================================== # Combined optionals and positionals tests # ======================================== class TestOptionalsNumericAndPositionals(ParserTestCase): """Tests negative number args when numeric options are present""" argument_signatures = [ Sig('x', nargs='?'), Sig('-4', dest='y', action='store_true'), ] failures = ['-2', '-315'] successes = [ ('', NS(x=None, y=False)), ('a', NS(x='a', y=False)), ('-4', NS(x=None, y=True)), ('-4 a', NS(x='a', y=True)), ] class TestOptionalsAlmostNumericAndPositionals(ParserTestCase): """Tests negative number args when almost numeric options are present""" argument_signatures = [ Sig('x', nargs='?'), Sig('-k4', dest='y', action='store_true'), ] failures = ['-k3'] successes = [ ('', NS(x=None, y=False)), ('-2', NS(x='-2', y=False)), ('a', NS(x='a', y=False)), ('-k4', NS(x=None, y=True)), ('-k4 a', NS(x='a', y=True)), ] class TestEmptyAndSpaceContainingArguments(ParserTestCase): argument_signatures = [ Sig('x', nargs='?'), Sig('-y', '--yyy', dest='y'), ] failures = ['-y'] successes = [ ([''], NS(x='', y=None)), (['a badger'], NS(x='a badger', y=None)), (['-a badger'], NS(x='-a badger', y=None)), (['-y', ''], NS(x=None, y='')), (['-y', 'a badger'], NS(x=None, y='a badger')), (['-y', '-a badger'], NS(x=None, y='-a badger')), (['--yyy=a badger'], NS(x=None, y='a badger')), (['--yyy=-a badger'], NS(x=None, y='-a badger')), ] class TestPrefixCharacterOnlyArguments(ParserTestCase): parser_signature = Sig(prefix_chars='-+') argument_signatures = [ Sig('-', dest='x', nargs='?', const='badger'), Sig('+', dest='y', type=int, default=42), Sig('-+-', dest='z', action='store_true'), ] failures = ['-y', '+ -'] successes = [ ('', NS(x=None, y=42, z=False)), ('-', NS(x='badger', y=42, z=False)), ('- X', NS(x='X', y=42, z=False)), ('+ -3', NS(x=None, y=-3, z=False)), ('-+-', NS(x=None, y=42, z=True)), ('- ===', NS(x='===', y=42, z=False)), ] class TestNargsZeroOrMore(ParserTestCase): """Tests specifying args for an Optional that accepts zero or more""" argument_signatures = [Sig('-x', nargs='*'), Sig('y', nargs='*')] failures = [] successes = [ ('', NS(x=None, y=[])), ('-x', NS(x=[], y=[])), ('-x a', NS(x=['a'], y=[])), ('-x a -- b', NS(x=['a'], y=['b'])), ('a', NS(x=None, y=['a'])), ('a -x', NS(x=[], y=['a'])), ('a -x b', NS(x=['b'], y=['a'])), ] class TestNargsRemainder(ParserTestCase): """Tests specifying a positional with nargs=REMAINDER""" argument_signatures = [Sig('x'), Sig('y', nargs='...'), Sig('-z')] failures = ['', '-z', '-z Z'] successes = [ ('X', NS(x='X', y=[], z=None)), ('-z Z X', NS(x='X', y=[], z='Z')), ('X A B -z Z', NS(x='X', y=['A', 'B', '-z', 'Z'], z=None)), ('X Y --foo', NS(x='X', y=['Y', '--foo'], z=None)), ] class TestOptionLike(ParserTestCase): """Tests options that may or may not be arguments""" argument_signatures = [ Sig('-x', type=float), Sig('-3', type=float, dest='y'), Sig('z', nargs='*'), ] failures = ['-x', '-y2.5', '-xa', '-x -a', '-x -3', '-x -3.5', '-3 -3.5', '-x -2.5', '-x -2.5 a', '-3 -.5', 'a x -1', '-x -1 a', '-3 -1 a'] successes = [ ('', NS(x=None, y=None, z=[])), ('-x 2.5', NS(x=2.5, y=None, z=[])), ('-x 2.5 a', NS(x=2.5, y=None, z=['a'])), ('-3.5', NS(x=None, y=0.5, z=[])), ('-3-.5', NS(x=None, y=-0.5, z=[])), ('-3 .5', NS(x=None, y=0.5, z=[])), ('a -3.5', NS(x=None, y=0.5, z=['a'])), ('a', NS(x=None, y=None, z=['a'])), ('a -x 1', NS(x=1.0, y=None, z=['a'])), ('-x 1 a', NS(x=1.0, y=None, z=['a'])), ('-3 1 a', NS(x=None, y=1.0, z=['a'])), ] class TestDefaultSuppress(ParserTestCase): """Test actions with suppressed defaults""" argument_signatures = [ Sig('foo', nargs='?', default=argparse.SUPPRESS), Sig('bar', nargs='*', default=argparse.SUPPRESS), Sig('--baz', action='store_true', default=argparse.SUPPRESS), ] failures = ['-x'] successes = [ ('', NS()), ('a', NS(foo='a')), ('a b', NS(foo='a', bar=['b'])), ('--baz', NS(baz=True)), ('a --baz', NS(foo='a', baz=True)), ('--baz a b', NS(foo='a', bar=['b'], baz=True)), ] class TestParserDefaultSuppress(ParserTestCase): """Test actions with a parser-level default of SUPPRESS""" parser_signature = Sig(argument_default=argparse.SUPPRESS) argument_signatures = [ Sig('foo', nargs='?'), Sig('bar', nargs='*'), Sig('--baz', action='store_true'), ] failures = ['-x'] successes = [ ('', NS()), ('a', NS(foo='a')), ('a b', NS(foo='a', bar=['b'])), ('--baz', NS(baz=True)), ('a --baz', NS(foo='a', baz=True)), ('--baz a b', NS(foo='a', bar=['b'], baz=True)), ] class TestParserDefault42(ParserTestCase): """Test actions with a parser-level default of 42""" parser_signature = Sig(argument_default=42, version='1.0') argument_signatures = [ Sig('foo', nargs='?'), Sig('bar', nargs='*'), Sig('--baz', action='store_true'), ] failures = ['-x'] successes = [ ('', NS(foo=42, bar=42, baz=42)), ('a', NS(foo='a', bar=42, baz=42)), ('a b', NS(foo='a', bar=['b'], baz=42)), ('--baz', NS(foo=42, bar=42, baz=True)), ('a --baz', NS(foo='a', bar=42, baz=True)), ('--baz a b', NS(foo='a', bar=['b'], baz=True)), ] # class TestArgumentsFromFile(TempDirMixin, ParserTestCase): # """Test reading arguments from a file""" # def setUp(self): # super(TestArgumentsFromFile, self).setUp() # file_texts = [ # ('hello', 'hello world!\n'), # ('recursive', '-a\n' # 'A\n' # '@hello'), # ('invalid', '@no-such-path\n'), # ] # for path, text in file_texts: # file = open(path, 'w') # file.write(text) # file.close() # parser_signature = Sig(fromfile_prefix_chars='@') # argument_signatures = [ # Sig('-a'), # Sig('x'), # Sig('y', nargs='+'), # ] # failures = ['', '-b', 'X', '@invalid', '@missing'] # successes = [ # ('X Y', NS(a=None, x='X', y=['Y'])), # ('X -a A Y Z', NS(a='A', x='X', y=['Y', 'Z'])), # ('@hello X', NS(a=None, x='hello world!', y=['X'])), # ('X @hello', NS(a=None, x='X', y=['hello world!'])), # ('-a B @recursive Y Z', NS(a='A', x='hello world!', y=['Y', 'Z'])), # ('X @recursive Z -a B', NS(a='B', x='X', y=['hello world!', 'Z'])), # (["-a", "", "X", "Y"], NS(a='', x='X', y=['Y'])), # ] # class TestArgumentsFromFileConverter(TempDirMixin, ParserTestCase): # """Test reading arguments from a file""" # def setUp(self): # super(TestArgumentsFromFileConverter, self).setUp() # file_texts = [ # ('hello', 'hello world!\n'), # ] # for path, text in file_texts: # file = open(path, 'w') # file.write(text) # file.close() # class FromFileConverterArgumentParser(ErrorRaisingArgumentParser): # def convert_arg_line_to_args(self, arg_line): # for arg in arg_line.split(): # if not arg.strip(): # continue # yield arg # parser_class = FromFileConverterArgumentParser # parser_signature = Sig(fromfile_prefix_chars='@') # argument_signatures = [ # Sig('y', nargs='+'), # ] # failures = [] # successes = [ # ('@hello X', NS(y=['hello', 'world!', 'X'])), # ] # ===================== # Type conversion tests # ===================== class TestFileTypeRepr(TestCase): def test_r(self): type = argparse.FileType('r') self.assertEqual("FileType('r')", repr(type)) def test_wb_1(self): type = argparse.FileType('wb', 1) self.assertEqual("FileType('wb', 1)", repr(type)) class RFile(object): seen = {} def __init__(self, name): self.name = name __hash__ = None def __eq__(self, other): if other in self.seen: text = self.seen[other] else: text = self.seen[other] = other.read() other.close() if not isinstance(text, str): text = text.decode('ascii') return self.name == other.name == text # class TestFileTypeR(TempDirMixin, ParserTestCase): # """Test the FileType option/argument type for reading files""" # def setUp(self): # super(TestFileTypeR, self).setUp() # for file_name in ['foo', 'bar']: # file = open(os.path.join(self.temp_dir, file_name), 'w') # file.write(file_name) # file.close() # self.create_readonly_file('readonly') # argument_signatures = [ # Sig('-x', type=argparse.FileType()), # Sig('spam', type=argparse.FileType('r')), # ] # failures = ['-x', '-x bar', 'non-existent-file.txt'] # successes = [ # ('foo', NS(x=None, spam=RFile('foo'))), # ('-x foo bar', NS(x=RFile('foo'), spam=RFile('bar'))), # ('bar -x foo', NS(x=RFile('foo'), spam=RFile('bar'))), # ('-x - -', NS(x=sys.stdin, spam=sys.stdin)), # ('readonly', NS(x=None, spam=RFile('readonly'))), # ] # class TestFileTypeDefaults(TempDirMixin, ParserTestCase): # """Test that a file is not created unless the default is needed""" # def setUp(self): # super(TestFileTypeDefaults, self).setUp() # file = open(os.path.join(self.temp_dir, 'good'), 'w') # file.write('good') # file.close() # argument_signatures = [ # Sig('-c', type=argparse.FileType('r'), default='no-file.txt'), # ] # # should provoke no such file error # failures = [''] # # should not provoke error because default file is created # successes = [('-c good', NS(c=RFile('good')))] # class TestFileTypeRB(TempDirMixin, ParserTestCase): # """Test the FileType option/argument type for reading files""" # def setUp(self): # super(TestFileTypeRB, self).setUp() # for file_name in ['foo', 'bar']: # file = open(os.path.join(self.temp_dir, file_name), 'w') # file.write(file_name) # file.close() # argument_signatures = [ # Sig('-x', type=argparse.FileType('rb')), # Sig('spam', type=argparse.FileType('rb')), # ] # failures = ['-x', '-x bar'] # successes = [ # ('foo', NS(x=None, spam=RFile('foo'))), # ('-x foo bar', NS(x=RFile('foo'), spam=RFile('bar'))), # ('bar -x foo', NS(x=RFile('foo'), spam=RFile('bar'))), # ('-x - -', NS(x=sys.stdin, spam=sys.stdin)), # ] class WFile(object): seen = set() def __init__(self, name): self.name = name __hash__ = None def __eq__(self, other): if other not in self.seen: text = 'Check that file is writable.' if 'b' in other.mode: text = text.encode('ascii') other.write(text) other.close() self.seen.add(other) return self.name == other.name # @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, # "non-root user required") # class TestFileTypeW(TempDirMixin, ParserTestCase): # """Test the FileType option/argument type for writing files""" # def setUp(self): # super(TestFileTypeW, self).setUp() # self.create_readonly_file('readonly') # argument_signatures = [ # Sig('-x', type=argparse.FileType('w')), # Sig('spam', type=argparse.FileType('w')), # ] # failures = ['-x', '-x bar'] # failures = ['-x', '-x bar', 'readonly'] # successes = [ # ('foo', NS(x=None, spam=WFile('foo'))), # ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), # ('bar -x foo', NS(x=WFile('foo'), spam=WFile('bar'))), # ('-x - -', NS(x=sys.stdout, spam=sys.stdout)), # ] # class TestFileTypeWB(TempDirMixin, ParserTestCase): # argument_signatures = [ # Sig('-x', type=argparse.FileType('wb')), # Sig('spam', type=argparse.FileType('wb')), # ] # failures = ['-x', '-x bar'] # successes = [ # ('foo', NS(x=None, spam=WFile('foo'))), # ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), # ('bar -x foo', NS(x=WFile('foo'), spam=WFile('bar'))), # ('-x - -', NS(x=sys.stdout, spam=sys.stdout)), # ] # class TestTypeCallable(ParserTestCase): # """Test some callables as option/argument types""" # argument_signatures = [ # Sig('--eggs', type=complex), # Sig('spam', type=float), # ] # failures = ['a', '42j', '--eggs a', '--eggs 2i'] # successes = [ # ('--eggs=42 42', NS(eggs=42, spam=42.0)), # ('--eggs 2j -- -1.5', NS(eggs=2j, spam=-1.5)), # ('1024.675', NS(eggs=None, spam=1024.675)), # ] class TestTypeUserDefined(ParserTestCase): """Test a user-defined option/argument type""" class MyType(TestCase): def __init__(self, value): self.value = value __hash__ = None def __eq__(self, other): return (type(self), self.value) == (type(other), other.value) argument_signatures = [ Sig('-x', type=MyType), Sig('spam', type=MyType), ] failures = [] successes = [ ('a -x b', NS(x=MyType('b'), spam=MyType('a'))), ('-xf g', NS(x=MyType('f'), spam=MyType('g'))), ] class TestTypeClassicClass(ParserTestCase): """Test a classic class type""" class C(object): def __init__(self, value): self.value = value __hash__ = None def __eq__(self, other): return (type(self), self.value) == (type(other), other.value) argument_signatures = [ Sig('-x', type=C), Sig('spam', type=C), ] failures = [] successes = [ ('a -x b', NS(x=C('b'), spam=C('a'))), ('-xf g', NS(x=C('f'), spam=C('g'))), ] class TestTypeRegistration(TestCase): """Test a user-defined type by registering it""" def test(self): def get_my_type(string): return 'my_type{%s}' % string parser = argparse.ArgumentParser() parser.register('type', 'my_type', get_my_type) parser.add_argument('-x', type='my_type') parser.add_argument('y', type='my_type') self.assertEqual(parser.parse_args('1'.split()), NS(x=None, y='my_type{1}')) self.assertEqual(parser.parse_args('-x 1 42'.split()), NS(x='my_type{1}', y='my_type{42}')) # ============ # Action tests # ============ class TestActionUserDefined(ParserTestCase): """Test a user-defined option/argument action""" class OptionalAction(argparse.Action): def __call__(self, parser, namespace, value, option_string=None): try: # check destination and option string assert self.dest == 'spam', 'dest: %s' % self.dest assert option_string == '-s', 'flag: %s' % option_string # when option is before argument, badger=2, and when # option is after argument, badger= expected_ns = NS(spam=0.25) if value in [0.125, 0.625]: expected_ns.badger = 2 elif value in [2.0]: expected_ns.badger = 84 else: raise AssertionError('value: %s' % value) assert expected_ns == namespace, ('expected %s, got %s' % (expected_ns, namespace)) except AssertionError: e = sys.exc_info()[1] raise ArgumentParserError('opt_action failed: %s' % e) setattr(namespace, 'spam', value) class PositionalAction(argparse.Action): def __call__(self, parser, namespace, value, option_string=None): try: assert option_string is None, ('option_string: %s' % option_string) # check destination assert self.dest == 'badger', 'dest: %s' % self.dest # when argument is before option, spam=0.25, and when # option is after argument, spam= expected_ns = NS(badger=2) if value in [42, 84]: expected_ns.spam = 0.25 elif value in [1]: expected_ns.spam = 0.625 elif value in [2]: expected_ns.spam = 0.125 else: raise AssertionError('value: %s' % value) assert expected_ns == namespace, ('expected %s, got %s' % (expected_ns, namespace)) except AssertionError: e = sys.exc_info()[1] raise ArgumentParserError('arg_action failed: %s' % e) setattr(namespace, 'badger', value) argument_signatures = [ Sig('-s', dest='spam', action=OptionalAction, type=float, default=0.25), Sig('badger', action=PositionalAction, type=int, nargs='?', default=2), ] failures = [] successes = [ ('-s0.125', NS(spam=0.125, badger=2)), ('42', NS(spam=0.25, badger=42)), ('-s 0.625 1', NS(spam=0.625, badger=1)), ('84 -s2', NS(spam=2.0, badger=84)), ] class TestActionRegistration(TestCase): """Test a user-defined action supplied by registering it""" class MyAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, 'foo[%s]' % values) def test(self): parser = argparse.ArgumentParser() parser.register('action', 'my_action', self.MyAction) parser.add_argument('badger', action='my_action') self.assertEqual(parser.parse_args(['1']), NS(badger='foo[1]')) self.assertEqual(parser.parse_args(['42']), NS(badger='foo[42]')) # ================ # Subparsers tests # ================ class TestAddSubparsers(TestCase): """Test the add_subparsers method""" def assertArgumentParserError(self, *args, **kwargs): self.assertRaises(ArgumentParserError, *args, **kwargs) def _get_parser(self, subparser_help=False, prefix_chars=None): # create a parser with a subparsers argument if prefix_chars: parser = ErrorRaisingArgumentParser( prog='PROG', description='main description', prefix_chars=prefix_chars) parser.add_argument( prefix_chars[0] * 2 + 'foo', action='store_true', help='foo help') else: parser = ErrorRaisingArgumentParser( prog='PROG', description='main description') parser.add_argument( '--foo', action='store_true', help='foo help') parser.add_argument( 'bar', type=float, help='bar help') # check that only one subparsers argument can be added subparsers = parser.add_subparsers(help='command help') self.assertArgumentParserError(parser.add_subparsers) # add first sub-parser parser1_kwargs = dict(description='1 description') if subparser_help: parser1_kwargs['help'] = '1 help' parser1 = subparsers.add_parser('1', **parser1_kwargs) parser1.add_argument('-w', type=int, help='w help') parser1.add_argument('x', choices='abc', help='x help') # add second sub-parser parser2_kwargs = dict(description='2 description') if subparser_help: parser2_kwargs['help'] = '2 help' parser2 = subparsers.add_parser('2', **parser2_kwargs) parser2.add_argument('-y', choices='123', help='y help') # parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser parser3_kwargs = dict(description='3 description') if subparser_help: parser3_kwargs['help'] = '3 help' parser3 = subparsers.add_parser('3', **parser3_kwargs) parser3.add_argument('t', type=int, help='t help') parser3.add_argument('u', nargs='...', help='u help') # return the main parser return parser def setUp(self): super(TestAddSubparsers, self).setUp() self.parser = self._get_parser() self.command_help_parser = self._get_parser(subparser_help=True) def test_parse_args_failures(self): # check some failure cases: for args_str in ['', 'a', 'a a', '0.5 a', '0.5 1', '0.5 1 -y', '0.5 2 -w']: args = args_str.split() self.assertArgumentParserError(self.parser.parse_args, args) def test_parse_args(self): # check some non-failure cases: self.assertEqual( self.parser.parse_args('0.5 1 b -w 7'.split()), NS(foo=False, bar=0.5, w=7, x='b'), ) # self.assertEqual( # self.parser.parse_args('0.25 --foo 2 -y 2 3j -- -1j'.split()), # NS(foo=True, bar=0.25, y='2', z=[3j, -1j]), # ) self.assertEqual( self.parser.parse_args('--foo 0.125 1 c'.split()), NS(foo=True, bar=0.125, w=None, x='c'), ) self.assertEqual( self.parser.parse_args('-1.5 3 11 -- a --foo 7 -- b'.split()), NS(foo=False, bar=-1.5, t=11, u=['a', '--foo', '7', '--', 'b']), ) def test_parse_known_args(self): self.assertEqual( self.parser.parse_known_args('0.5 1 b -w 7'.split()), (NS(foo=False, bar=0.5, w=7, x='b'), []), ) self.assertEqual( self.parser.parse_known_args('0.5 -p 1 b -w 7'.split()), (NS(foo=False, bar=0.5, w=7, x='b'), ['-p']), ) self.assertEqual( self.parser.parse_known_args('0.5 1 b -w 7 -p'.split()), (NS(foo=False, bar=0.5, w=7, x='b'), ['-p']), ) self.assertEqual( self.parser.parse_known_args('0.5 1 b -q -rs -w 7'.split()), (NS(foo=False, bar=0.5, w=7, x='b'), ['-q', '-rs']), ) self.assertEqual( self.parser.parse_known_args('0.5 -W 1 b -X Y -w 7 Z'.split()), (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) def test_dest(self): parser = ErrorRaisingArgumentParser() parser.add_argument('--foo', action='store_true') subparsers = parser.add_subparsers(dest='bar') parser1 = subparsers.add_parser('1') parser1.add_argument('baz') self.assertEqual(NS(foo=False, bar='1', baz='2'), parser.parse_args('1 2'.split())) def test_help(self): self.assertEqual(self.parser.format_usage(), 'usage: PROG [-h] [--foo] bar {1,2,3} ...\n') self.assertEqual(self.parser.format_help(), textwrap.dedent('''\ usage: PROG [-h] [--foo] bar {1,2,3} ... main description positional arguments: bar bar help {1,2,3} command help optional arguments: -h, --help show this help message and exit --foo foo help ''')) def test_help_extra_prefix_chars(self): # Make sure - is still used for help if it is a non-first prefix char parser = self._get_parser(prefix_chars='+:-') self.assertEqual(parser.format_usage(), 'usage: PROG [-h] [++foo] bar {1,2,3} ...\n') self.assertEqual(parser.format_help(), textwrap.dedent('''\ usage: PROG [-h] [++foo] bar {1,2,3} ... main description positional arguments: bar bar help {1,2,3} command help optional arguments: -h, --help show this help message and exit ++foo foo help ''')) def test_help_alternate_prefix_chars(self): parser = self._get_parser(prefix_chars='+:/') self.assertEqual(parser.format_usage(), 'usage: PROG [+h] [++foo] bar {1,2,3} ...\n') self.assertEqual(parser.format_help(), textwrap.dedent('''\ usage: PROG [+h] [++foo] bar {1,2,3} ... main description positional arguments: bar bar help {1,2,3} command help optional arguments: +h, ++help show this help message and exit ++foo foo help ''')) def test_parser_command_help(self): self.assertEqual(self.command_help_parser.format_usage(), 'usage: PROG [-h] [--foo] bar {1,2,3} ...\n') self.assertEqual(self.command_help_parser.format_help(), textwrap.dedent('''\ usage: PROG [-h] [--foo] bar {1,2,3} ... main description positional arguments: bar bar help {1,2,3} command help 1 1 help 2 2 help 3 3 help optional arguments: -h, --help show this help message and exit --foo foo help ''')) def test_subparser_title_help(self): parser = ErrorRaisingArgumentParser(prog='PROG', description='main description') parser.add_argument('--foo', action='store_true', help='foo help') parser.add_argument('bar', help='bar help') subparsers = parser.add_subparsers(title='subcommands', description='command help', help='additional text') parser1 = subparsers.add_parser('1') parser2 = subparsers.add_parser('2') self.assertEqual(parser.format_usage(), 'usage: PROG [-h] [--foo] bar {1,2} ...\n') self.assertEqual(parser.format_help(), textwrap.dedent('''\ usage: PROG [-h] [--foo] bar {1,2} ... main description positional arguments: bar bar help optional arguments: -h, --help show this help message and exit --foo foo help subcommands: command help {1,2} additional text ''')) def _test_subparser_help(self, args_str, expected_help): try: self.parser.parse_args(args_str.split()) except ArgumentParserError: err = sys.exc_info()[1] if err.stdout != expected_help: print(repr(expected_help)) print(repr(err.stdout)) self.assertEqual(err.stdout, expected_help) def test_subparser1_help(self): self._test_subparser_help('5.0 1 -h', textwrap.dedent('''\ usage: PROG bar 1 [-h] [-w W] {a,b,c} 1 description positional arguments: {a,b,c} x help optional arguments: -h, --help show this help message and exit -w W w help ''')) @unittest.expectedFailure def test_subparser2_help(self): self._test_subparser_help('5.0 2 -h', textwrap.dedent('''\ usage: PROG bar 2 [-h] [-y {1,2,3}] [z [z ...]] 2 description positional arguments: z z help optional arguments: -h, --help show this help message and exit -y {1,2,3} y help ''')) # ============ # Groups tests # ============ class TestPositionalsGroups(TestCase): """Tests that order of group positionals matches construction order""" def test_nongroup_first(self): parser = ErrorRaisingArgumentParser() parser.add_argument('foo') group = parser.add_argument_group('g') group.add_argument('bar') parser.add_argument('baz') expected = NS(foo='1', bar='2', baz='3') result = parser.parse_args('1 2 3'.split()) self.assertEqual(expected, result) def test_group_first(self): parser = ErrorRaisingArgumentParser() group = parser.add_argument_group('xxx') group.add_argument('foo') parser.add_argument('bar') parser.add_argument('baz') expected = NS(foo='1', bar='2', baz='3') result = parser.parse_args('1 2 3'.split()) self.assertEqual(expected, result) def test_interleaved_groups(self): parser = ErrorRaisingArgumentParser() group = parser.add_argument_group('xxx') parser.add_argument('foo') group.add_argument('bar') parser.add_argument('baz') group = parser.add_argument_group('yyy') group.add_argument('frell') expected = NS(foo='1', bar='2', baz='3', frell='4') result = parser.parse_args('1 2 3 4'.split()) self.assertEqual(expected, result) # =================== # Parent parser tests # =================== class TestParentParsers(TestCase): """Tests that parsers can be created with parent parsers""" def assertArgumentParserError(self, *args, **kwargs): self.assertRaises(ArgumentParserError, *args, **kwargs) def setUp(self): super(TestParentParsers, self).setUp() self.wxyz_parent = ErrorRaisingArgumentParser(add_help=False) self.wxyz_parent.add_argument('--w') x_group = self.wxyz_parent.add_argument_group('x') x_group.add_argument('-y') self.wxyz_parent.add_argument('z') self.abcd_parent = ErrorRaisingArgumentParser(add_help=False) self.abcd_parent.add_argument('a') self.abcd_parent.add_argument('-b') c_group = self.abcd_parent.add_argument_group('c') c_group.add_argument('--d') self.w_parent = ErrorRaisingArgumentParser(add_help=False) self.w_parent.add_argument('--w') self.z_parent = ErrorRaisingArgumentParser(add_help=False) self.z_parent.add_argument('z') # parents with mutually exclusive groups self.ab_mutex_parent = ErrorRaisingArgumentParser(add_help=False) group = self.ab_mutex_parent.add_mutually_exclusive_group() group.add_argument('-a', action='store_true') group.add_argument('-b', action='store_true') self.main_program = os.path.basename(sys.argv[0]) def test_single_parent(self): parser = ErrorRaisingArgumentParser(parents=[self.wxyz_parent]) self.assertEqual(parser.parse_args('-y 1 2 --w 3'.split()), NS(w='3', y='1', z='2')) def test_single_parent_mutex(self): self._test_mutex_ab(self.ab_mutex_parent.parse_args) parser = ErrorRaisingArgumentParser(parents=[self.ab_mutex_parent]) self._test_mutex_ab(parser.parse_args) def test_single_granparent_mutex(self): parents = [self.ab_mutex_parent] parser = ErrorRaisingArgumentParser(add_help=False, parents=parents) parser = ErrorRaisingArgumentParser(parents=[parser]) self._test_mutex_ab(parser.parse_args) def _test_mutex_ab(self, parse_args): self.assertEqual(parse_args([]), NS(a=False, b=False)) self.assertEqual(parse_args(['-a']), NS(a=True, b=False)) self.assertEqual(parse_args(['-b']), NS(a=False, b=True)) self.assertArgumentParserError(parse_args, ['-a', '-b']) self.assertArgumentParserError(parse_args, ['-b', '-a']) self.assertArgumentParserError(parse_args, ['-c']) self.assertArgumentParserError(parse_args, ['-a', '-c']) self.assertArgumentParserError(parse_args, ['-b', '-c']) def test_multiple_parents(self): parents = [self.abcd_parent, self.wxyz_parent] parser = ErrorRaisingArgumentParser(parents=parents) self.assertEqual(parser.parse_args('--d 1 --w 2 3 4'.split()), NS(a='3', b=None, d='1', w='2', y=None, z='4')) def test_multiple_parents_mutex(self): parents = [self.ab_mutex_parent, self.wxyz_parent] parser = ErrorRaisingArgumentParser(parents=parents) self.assertEqual(parser.parse_args('-a --w 2 3'.split()), NS(a=True, b=False, w='2', y=None, z='3')) self.assertArgumentParserError( parser.parse_args, '-a --w 2 3 -b'.split()) self.assertArgumentParserError( parser.parse_args, '-a -b --w 2 3'.split()) def test_conflicting_parents(self): self.assertRaises( argparse.ArgumentError, argparse.ArgumentParser, parents=[self.w_parent, self.wxyz_parent]) def test_conflicting_parents_mutex(self): self.assertRaises( argparse.ArgumentError, argparse.ArgumentParser, parents=[self.abcd_parent, self.ab_mutex_parent]) def test_same_argument_name_parents(self): parents = [self.wxyz_parent, self.z_parent] parser = ErrorRaisingArgumentParser(parents=parents) self.assertEqual(parser.parse_args('1 2'.split()), NS(w=None, y=None, z='2')) def test_subparser_parents(self): parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers() abcde_parser = subparsers.add_parser('bar', parents=[self.abcd_parent]) abcde_parser.add_argument('e') self.assertEqual(parser.parse_args('bar -b 1 --d 2 3 4'.split()), NS(a='3', b='1', d='2', e='4')) def test_subparser_parents_mutex(self): parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers() parents = [self.ab_mutex_parent] abc_parser = subparsers.add_parser('foo', parents=parents) c_group = abc_parser.add_argument_group('c_group') c_group.add_argument('c') parents = [self.wxyz_parent, self.ab_mutex_parent] wxyzabe_parser = subparsers.add_parser('bar', parents=parents) wxyzabe_parser.add_argument('e') self.assertEqual(parser.parse_args('foo -a 4'.split()), NS(a=True, b=False, c='4')) self.assertEqual(parser.parse_args('bar -b --w 2 3 4'.split()), NS(a=False, b=True, w='2', y=None, z='3', e='4')) self.assertArgumentParserError( parser.parse_args, 'foo -a -b 4'.split()) self.assertArgumentParserError( parser.parse_args, 'bar -b -a 4'.split()) def test_parent_help(self): parents = [self.abcd_parent, self.wxyz_parent] parser = ErrorRaisingArgumentParser(parents=parents) parser_help = parser.format_help() progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ usage: %s[-h] [-b B] [--d D] [--w W] [-y Y] a z positional arguments: a z optional arguments: -h, --help show this help message and exit -b B --w W c: --d D x: -y Y ''' % (progname + ' ' if progname else ''))) @unittest.expectedFailure def test_groups_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) g = parent.add_argument_group(title='g', description='gd') g.add_argument('-w') g.add_argument('-x') m = parent.add_mutually_exclusive_group() m.add_argument('-y') m.add_argument('-z') parser = ErrorRaisingArgumentParser(parents=[parent]) self.assertRaises(ArgumentParserError, parser.parse_args, ['-y', 'Y', '-z', 'Z']) parser_help = parser.format_help() progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ usage: %s[-h] [-w W] [-x X] [-y Y | -z Z] optional arguments: -h, --help show this help message and exit -y Y -z Z g: gd -w W -x X ''' % (progname + ' ' if progname else '' ))) # ============================== # Mutually exclusive group tests # ============================== class TestMutuallyExclusiveGroupErrors(TestCase): def test_invalid_add_argument_group(self): parser = ErrorRaisingArgumentParser() raises = self.assertRaises raises(TypeError, parser.add_mutually_exclusive_group, title='foo') def test_invalid_add_argument(self): parser = ErrorRaisingArgumentParser() group = parser.add_mutually_exclusive_group() add_argument = group.add_argument raises = self.assertRaises raises(ValueError, add_argument, '--foo', required=True) raises(ValueError, add_argument, 'bar') raises(ValueError, add_argument, 'bar', nargs='+') raises(ValueError, add_argument, 'bar', nargs=1) raises(ValueError, add_argument, 'bar', nargs=argparse.PARSER) @unittest.expectedFailure def test_help(self): parser = ErrorRaisingArgumentParser(prog='PROG') group1 = parser.add_mutually_exclusive_group() group1.add_argument('--foo', action='store_true') group1.add_argument('--bar', action='store_false') group2 = parser.add_mutually_exclusive_group() group2.add_argument('--soup', action='store_true') group2.add_argument('--nuts', action='store_false') expected = '''\ usage: PROG [-h] [--foo | --bar] [--soup | --nuts] optional arguments: -h, --help show this help message and exit --foo --bar --soup --nuts ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) class MEMixin(object): def test_failures_when_not_required(self): parse_args = self.get_parser(required=False).parse_args error = ArgumentParserError for args_string in self.failures: self.assertRaises(error, parse_args, args_string.split()) def test_failures_when_required(self): parse_args = self.get_parser(required=True).parse_args error = ArgumentParserError for args_string in self.failures + ['']: self.assertRaises(error, parse_args, args_string.split()) def test_successes_when_not_required(self): parse_args = self.get_parser(required=False).parse_args successes = self.successes + self.successes_when_not_required for args_string, expected_ns in successes: actual_ns = parse_args(args_string.split()) self.assertEqual(actual_ns, expected_ns) def test_successes_when_required(self): parse_args = self.get_parser(required=True).parse_args for args_string, expected_ns in self.successes: actual_ns = parse_args(args_string.split()) self.assertEqual(actual_ns, expected_ns) def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage expected_usage = self.usage_when_not_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) def test_usage_when_required(self): format_usage = self.get_parser(required=True).format_usage expected_usage = self.usage_when_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) def test_help_when_not_required(self): format_help = self.get_parser(required=False).format_help help = self.usage_when_not_required + self.help self.assertEqual(format_help(), textwrap.dedent(help)) def test_help_when_required(self): format_help = self.get_parser(required=True).format_help help = self.usage_when_required + self.help self.assertEqual(format_help(), textwrap.dedent(help)) class TestMutuallyExclusiveSimple(MEMixin, TestCase): def get_parser(self, required=None): parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group(required=required) group.add_argument('--bar', help='bar help') group.add_argument('--baz', nargs='?', const='Z', help='baz help') return parser failures = ['--bar X --baz Y', '--bar X --baz'] successes = [ ('--bar X', NS(bar='X', baz=None)), ('--bar X --bar Z', NS(bar='Z', baz=None)), ('--baz Y', NS(bar=None, baz='Y')), ('--baz', NS(bar=None, baz='Z')), ] successes_when_not_required = [ ('', NS(bar=None, baz=None)), ] usage_when_not_required = '''\ usage: PROG [-h] [--bar BAR | --baz [BAZ]] ''' usage_when_required = '''\ usage: PROG [-h] (--bar BAR | --baz [BAZ]) ''' help = '''\ optional arguments: -h, --help show this help message and exit --bar BAR bar help --baz [BAZ] baz help ''' class TestMutuallyExclusiveLong(MEMixin, TestCase): def get_parser(self, required=None): parser = ErrorRaisingArgumentParser(prog='PROG') parser.add_argument('--abcde', help='abcde help') parser.add_argument('--fghij', help='fghij help') group = parser.add_mutually_exclusive_group(required=required) group.add_argument('--klmno', help='klmno help') group.add_argument('--pqrst', help='pqrst help') return parser failures = ['--klmno X --pqrst Y'] successes = [ ('--klmno X', NS(abcde=None, fghij=None, klmno='X', pqrst=None)), ('--abcde Y --klmno X', NS(abcde='Y', fghij=None, klmno='X', pqrst=None)), ('--pqrst X', NS(abcde=None, fghij=None, klmno=None, pqrst='X')), ('--pqrst X --fghij Y', NS(abcde=None, fghij='Y', klmno=None, pqrst='X')), ] successes_when_not_required = [ ('', NS(abcde=None, fghij=None, klmno=None, pqrst=None)), ] usage_when_not_required = '''\ usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] [--klmno KLMNO | --pqrst PQRST] ''' usage_when_required = '''\ usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] (--klmno KLMNO | --pqrst PQRST) ''' help = '''\ optional arguments: -h, --help show this help message and exit --abcde ABCDE abcde help --fghij FGHIJ fghij help --klmno KLMNO klmno help --pqrst PQRST pqrst help ''' class TestMutuallyExclusiveFirstSuppressed(MEMixin, TestCase): def get_parser(self, required): parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group(required=required) group.add_argument('-x', help=argparse.SUPPRESS) group.add_argument('-y', action='store_false', help='y help') return parser failures = ['-x X -y'] successes = [ ('-x X', NS(x='X', y=True)), ('-x X -x Y', NS(x='Y', y=True)), ('-y', NS(x=None, y=False)), ] successes_when_not_required = [ ('', NS(x=None, y=True)), ] usage_when_not_required = '''\ usage: PROG [-h] [-y] ''' usage_when_required = '''\ usage: PROG [-h] -y ''' help = '''\ optional arguments: -h, --help show this help message and exit -y y help ''' class TestMutuallyExclusiveManySuppressed(MEMixin, TestCase): def get_parser(self, required): parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group(required=required) add = group.add_argument add('--spam', action='store_true', help=argparse.SUPPRESS) add('--badger', action='store_false', help=argparse.SUPPRESS) add('--bladder', help=argparse.SUPPRESS) return parser failures = [ '--spam --badger', '--badger --bladder B', '--bladder B --spam', ] successes = [ ('--spam', NS(spam=True, badger=True, bladder=None)), ('--badger', NS(spam=False, badger=False, bladder=None)), ('--bladder B', NS(spam=False, badger=True, bladder='B')), ('--spam --spam', NS(spam=True, badger=True, bladder=None)), ] successes_when_not_required = [ ('', NS(spam=False, badger=True, bladder=None)), ] usage_when_required = usage_when_not_required = '''\ usage: PROG [-h] ''' help = '''\ optional arguments: -h, --help show this help message and exit ''' class TestMutuallyExclusiveOptionalAndPositional(MEMixin, TestCase): def get_parser(self, required): parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group(required=required) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') group.add_argument('badger', nargs='*', default='X', help='BADGER') return parser failures = [ '--foo --spam S', '--spam S X', 'X --foo', 'X Y Z --spam S', '--foo X Y', ] successes = [ ('--foo', NS(foo=True, spam=None, badger='X')), ('--spam S', NS(foo=False, spam='S', badger='X')), ('X', NS(foo=False, spam=None, badger=['X'])), ('X Y Z', NS(foo=False, spam=None, badger=['X', 'Y', 'Z'])), ] successes_when_not_required = [ ('', NS(foo=False, spam=None, badger='X')), ] usage_when_not_required = '''\ usage: PROG [-h] [--foo | --spam SPAM | badger [badger ...]] ''' usage_when_required = '''\ usage: PROG [-h] (--foo | --spam SPAM | badger [badger ...]) ''' help = '''\ positional arguments: badger BADGER optional arguments: -h, --help show this help message and exit --foo FOO --spam SPAM SPAM ''' class TestMutuallyExclusiveOptionalsMixed(MEMixin, TestCase): def get_parser(self, required): parser = ErrorRaisingArgumentParser(prog='PROG') parser.add_argument('-x', action='store_true', help='x help') group = parser.add_mutually_exclusive_group(required=required) group.add_argument('-a', action='store_true', help='a help') group.add_argument('-b', action='store_true', help='b help') parser.add_argument('-y', action='store_true', help='y help') group.add_argument('-c', action='store_true', help='c help') return parser failures = ['-a -b', '-b -c', '-a -c', '-a -b -c'] successes = [ ('-a', NS(a=True, b=False, c=False, x=False, y=False)), ('-b', NS(a=False, b=True, c=False, x=False, y=False)), ('-c', NS(a=False, b=False, c=True, x=False, y=False)), ('-a -x', NS(a=True, b=False, c=False, x=True, y=False)), ('-y -b', NS(a=False, b=True, c=False, x=False, y=True)), ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True)), ] successes_when_not_required = [ ('', NS(a=False, b=False, c=False, x=False, y=False)), ('-x', NS(a=False, b=False, c=False, x=True, y=False)), ('-y', NS(a=False, b=False, c=False, x=False, y=True)), ] usage_when_required = usage_when_not_required = '''\ usage: PROG [-h] [-x] [-a] [-b] [-y] [-c] ''' help = '''\ optional arguments: -h, --help show this help message and exit -x x help -a a help -b b help -y y help -c c help ''' class TestMutuallyExclusiveInGroup(MEMixin, TestCase): def get_parser(self, required=None): parser = ErrorRaisingArgumentParser(prog='PROG') titled_group = parser.add_argument_group( title='Titled group', description='Group description') mutex_group = \ titled_group.add_mutually_exclusive_group(required=required) mutex_group.add_argument('--bar', help='bar help') mutex_group.add_argument('--baz', help='baz help') return parser failures = ['--bar X --baz Y', '--baz X --bar Y'] successes = [ ('--bar X', NS(bar='X', baz=None)), ('--baz Y', NS(bar=None, baz='Y')), ] successes_when_not_required = [ ('', NS(bar=None, baz=None)), ] usage_when_not_required = '''\ usage: PROG [-h] [--bar BAR | --baz BAZ] ''' usage_when_required = '''\ usage: PROG [-h] (--bar BAR | --baz BAZ) ''' help = '''\ optional arguments: -h, --help show this help message and exit Titled group: Group description --bar BAR bar help --baz BAZ baz help ''' class TestMutuallyExclusiveOptionalsAndPositionalsMixed(MEMixin, TestCase): def get_parser(self, required): parser = ErrorRaisingArgumentParser(prog='PROG') parser.add_argument('x', help='x help') parser.add_argument('-y', action='store_true', help='y help') group = parser.add_mutually_exclusive_group(required=required) group.add_argument('a', nargs='?', help='a help') group.add_argument('-b', action='store_true', help='b help') group.add_argument('-c', action='store_true', help='c help') return parser failures = ['X A -b', '-b -c', '-c X A'] successes = [ ('X A', NS(a='A', b=False, c=False, x='X', y=False)), ('X -b', NS(a=None, b=True, c=False, x='X', y=False)), ('X -c', NS(a=None, b=False, c=True, x='X', y=False)), ('X A -y', NS(a='A', b=False, c=False, x='X', y=True)), ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True)), ] successes_when_not_required = [ ('X', NS(a=None, b=False, c=False, x='X', y=False)), ('X -y', NS(a=None, b=False, c=False, x='X', y=True)), ] usage_when_required = usage_when_not_required = '''\ usage: PROG [-h] [-y] [-b] [-c] x [a] ''' help = '''\ positional arguments: x x help a a help optional arguments: -h, --help show this help message and exit -y y help -b b help -c c help ''' # ================================================= # Mutually exclusive group in parent parser tests # ================================================= class MEPBase(object): def get_parser(self, required=None): parent = super(MEPBase, self).get_parser(required=required) parser = ErrorRaisingArgumentParser( prog=parent.prog, add_help=False, parents=[parent]) return parser class TestMutuallyExclusiveGroupErrorsParent( MEPBase, TestMutuallyExclusiveGroupErrors): pass class TestMutuallyExclusiveSimpleParent( MEPBase, TestMutuallyExclusiveSimple): pass class TestMutuallyExclusiveLongParent( MEPBase, TestMutuallyExclusiveLong): pass class TestMutuallyExclusiveFirstSuppressedParent( MEPBase, TestMutuallyExclusiveFirstSuppressed): pass class TestMutuallyExclusiveManySuppressedParent( MEPBase, TestMutuallyExclusiveManySuppressed): pass class TestMutuallyExclusiveOptionalAndPositionalParent( MEPBase, TestMutuallyExclusiveOptionalAndPositional): pass class TestMutuallyExclusiveOptionalsMixedParent( MEPBase, TestMutuallyExclusiveOptionalsMixed): pass class TestMutuallyExclusiveOptionalsAndPositionalsMixedParent( MEPBase, TestMutuallyExclusiveOptionalsAndPositionalsMixed): pass # ================= # Set default tests # ================= class TestSetDefaults(TestCase): def test_set_defaults_no_args(self): parser = ErrorRaisingArgumentParser() parser.set_defaults(x='foo') parser.set_defaults(y='bar', z=1) self.assertEqual(NS(x='foo', y='bar', z=1), parser.parse_args([])) self.assertEqual(NS(x='foo', y='bar', z=1), parser.parse_args([], NS())) self.assertEqual(NS(x='baz', y='bar', z=1), parser.parse_args([], NS(x='baz'))) self.assertEqual(NS(x='baz', y='bar', z=2), parser.parse_args([], NS(x='baz', z=2))) def test_set_defaults_with_args(self): parser = ErrorRaisingArgumentParser() parser.set_defaults(x='foo', y='bar') parser.add_argument('-x', default='xfoox') self.assertEqual(NS(x='xfoox', y='bar'), parser.parse_args([])) self.assertEqual(NS(x='xfoox', y='bar'), parser.parse_args([], NS())) self.assertEqual(NS(x='baz', y='bar'), parser.parse_args([], NS(x='baz'))) self.assertEqual(NS(x='1', y='bar'), parser.parse_args('-x 1'.split())) self.assertEqual(NS(x='1', y='bar'), parser.parse_args('-x 1'.split(), NS())) self.assertEqual(NS(x='1', y='bar'), parser.parse_args('-x 1'.split(), NS(x='baz'))) def test_set_defaults_subparsers(self): parser = ErrorRaisingArgumentParser() parser.set_defaults(x='foo') subparsers = parser.add_subparsers() parser_a = subparsers.add_parser('a') parser_a.set_defaults(y='bar') self.assertEqual(NS(x='foo', y='bar'), parser.parse_args('a'.split())) def test_set_defaults_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) parent.set_defaults(x='foo') parser = ErrorRaisingArgumentParser(parents=[parent]) self.assertEqual(NS(x='foo'), parser.parse_args([])) def test_set_defaults_on_parent_and_subparser(self): parser = argparse.ArgumentParser() xparser = parser.add_subparsers().add_parser('X') parser.set_defaults(foo=1) xparser.set_defaults(foo=2) self.assertEqual(NS(foo=2), parser.parse_args(['X'])) def test_set_defaults_same_as_add_argument(self): parser = ErrorRaisingArgumentParser() parser.set_defaults(w='W', x='X', y='Y', z='Z') parser.add_argument('-w') parser.add_argument('-x', default='XX') parser.add_argument('y', nargs='?') parser.add_argument('z', nargs='?', default='ZZ') # defaults set previously self.assertEqual(NS(w='W', x='XX', y='Y', z='ZZ'), parser.parse_args([])) # reset defaults parser.set_defaults(w='WW', x='X', y='YY', z='Z') self.assertEqual(NS(w='WW', x='X', y='YY', z='Z'), parser.parse_args([])) def test_set_defaults_same_as_add_argument_group(self): parser = ErrorRaisingArgumentParser() parser.set_defaults(w='W', x='X', y='Y', z='Z') group = parser.add_argument_group('foo') group.add_argument('-w') group.add_argument('-x', default='XX') group.add_argument('y', nargs='?') group.add_argument('z', nargs='?', default='ZZ') # defaults set previously self.assertEqual(NS(w='W', x='XX', y='Y', z='ZZ'), parser.parse_args([])) # reset defaults parser.set_defaults(w='WW', x='X', y='YY', z='Z') self.assertEqual(NS(w='WW', x='X', y='YY', z='Z'), parser.parse_args([])) # ================= # Get default tests # ================= class TestGetDefault(TestCase): def test_get_default(self): parser = ErrorRaisingArgumentParser() self.assertEqual(None, parser.get_default("foo")) self.assertEqual(None, parser.get_default("bar")) parser.add_argument("--foo") self.assertEqual(None, parser.get_default("foo")) self.assertEqual(None, parser.get_default("bar")) parser.add_argument("--bar", type=int, default=42) self.assertEqual(None, parser.get_default("foo")) self.assertEqual(42, parser.get_default("bar")) parser.set_defaults(foo="badger") self.assertEqual("badger", parser.get_default("foo")) self.assertEqual(42, parser.get_default("bar")) # ========================== # Namespace 'contains' tests # ========================== class TestNamespaceContainsSimple(TestCase): def test_empty(self): ns = argparse.Namespace() self.assertEqual('' in ns, False) self.assertEqual('' not in ns, True) self.assertEqual('x' in ns, False) def test_non_empty(self): ns = argparse.Namespace(x=1, y=2) self.assertEqual('x' in ns, True) self.assertEqual('x' not in ns, False) self.assertEqual('y' in ns, True) self.assertEqual('' in ns, False) self.assertEqual('xx' in ns, False) self.assertEqual('z' in ns, False) # ===================== # Help formatting tests # ===================== class TestHelpFormattingMetaclass(type): def __init__(cls, name, bases, bodydict): if name == 'HelpTestCase': return class AddTests(object): def __init__(self, test_class, func_suffix, std_name): self.func_suffix = func_suffix self.std_name = std_name for test_func in [self.test_format, self.test_print, self.test_print_file]: test_name = '%s_%s' % (test_func.__name__, func_suffix) def test_wrapper(self, test_func=test_func): test_func(self) # try: # test_wrapper.__name__ = test_name # except TypeError: # pass setattr(test_class, test_name, test_wrapper) def _get_parser(self, tester): parser = argparse.ArgumentParser( *tester.parser_signature.args, **tester.parser_signature.kwargs) for argument_sig in getattr(tester, 'argument_signatures', []): parser.add_argument(*argument_sig.args, **argument_sig.kwargs) group_sigs = getattr(tester, 'argument_group_signatures', []) for group_sig, argument_sigs in group_sigs: group = parser.add_argument_group(*group_sig.args, **group_sig.kwargs) for argument_sig in argument_sigs: group.add_argument(*argument_sig.args, **argument_sig.kwargs) subparsers_sigs = getattr(tester, 'subparsers_signatures', []) if subparsers_sigs: subparsers = parser.add_subparsers() for subparser_sig in subparsers_sigs: subparsers.add_parser(*subparser_sig.args, **subparser_sig.kwargs) return parser def _test(self, tester, parser_text): expected_text = getattr(tester, self.func_suffix) expected_text = textwrap.dedent(expected_text) if expected_text != parser_text: print(repr(expected_text)) print(repr(parser_text)) for char1, char2 in zip(expected_text, parser_text): if char1 != char2: print('first diff: %r %r' % (char1, char2)) break tester.assertEqual(expected_text, parser_text) def test_format(self, tester): parser = self._get_parser(tester) format = getattr(parser, 'format_%s' % self.func_suffix) self._test(tester, format()) def test_print(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) old_stream = getattr(sys, self.std_name) setattr(sys, self.std_name, StdIOBuffer()) try: print_() parser_text = getattr(sys, self.std_name).getvalue() finally: setattr(sys, self.std_name, old_stream) self._test(tester, parser_text) def test_print_file(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) sfile = StdIOBuffer() print_(sfile) parser_text = sfile.getvalue() self._test(tester, parser_text) # add tests for {format,print}_{usage,help,version} for func_suffix, std_name in [('usage', 'stdout'), ('help', 'stdout'), ('version', 'stderr')]: AddTests(cls, func_suffix, std_name) bases = TestCase, HelpTestCase = TestHelpFormattingMetaclass('HelpTestCase', bases, {}) class TestHelpBiggerOptionals(HelpTestCase): """Make sure that argument help aligns when options are longer""" parser_signature = Sig(prog='PROG', description='DESCRIPTION', epilog='EPILOG', version='0.1') argument_signatures = [ Sig('-x', action='store_true', help='X HELP'), Sig('--y', help='Y HELP'), Sig('foo', help='FOO HELP'), Sig('bar', help='BAR HELP'), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [-v] [-x] [--y Y] foo bar ''' help = usage + '''\ DESCRIPTION positional arguments: foo FOO HELP bar BAR HELP optional arguments: -h, --help show this help message and exit -v, --version show program's version number and exit -x X HELP --y Y Y HELP EPILOG ''' version = '''\ 0.1 ''' class TestShortColumns(HelpTestCase): '''Test extremely small number of columns. TestCase prevents "COLUMNS" from being too small in the tests themselves, but we don't want any exceptions thrown in such case. Only ugly representation. ''' def setUp(self): env = test_support.EnvironmentVarGuard() env.set("COLUMNS", '15') self.addCleanup(env.__exit__) parser_signature = TestHelpBiggerOptionals.parser_signature argument_signatures = TestHelpBiggerOptionals.argument_signatures argument_group_signatures = TestHelpBiggerOptionals.argument_group_signatures usage = '''\ usage: PROG [-h] [-v] [-x] [--y Y] foo bar ''' help = usage + '''\ DESCRIPTION positional arguments: foo FOO HELP bar BAR HELP optional arguments: -h, --help show this help message and exit -v, --version show program's version number and exit -x X HELP --y Y Y HELP EPILOG ''' version = TestHelpBiggerOptionals.version class TestHelpBiggerOptionalGroups(HelpTestCase): """Make sure that argument help aligns when options are longer""" parser_signature = Sig(prog='PROG', description='DESCRIPTION', epilog='EPILOG', version='0.1') argument_signatures = [ Sig('-x', action='store_true', help='X HELP'), Sig('--y', help='Y HELP'), Sig('foo', help='FOO HELP'), Sig('bar', help='BAR HELP'), ] argument_group_signatures = [ (Sig('GROUP TITLE', description='GROUP DESCRIPTION'), [ Sig('baz', help='BAZ HELP'), Sig('-z', nargs='+', help='Z HELP')]), ] usage = '''\ usage: PROG [-h] [-v] [-x] [--y Y] [-z Z [Z ...]] foo bar baz ''' help = usage + '''\ DESCRIPTION positional arguments: foo FOO HELP bar BAR HELP optional arguments: -h, --help show this help message and exit -v, --version show program's version number and exit -x X HELP --y Y Y HELP GROUP TITLE: GROUP DESCRIPTION baz BAZ HELP -z Z [Z ...] Z HELP EPILOG ''' version = '''\ 0.1 ''' class TestHelpBiggerPositionals(HelpTestCase): """Make sure that help aligns when arguments are longer""" parser_signature = Sig(usage='USAGE', description='DESCRIPTION') argument_signatures = [ Sig('-x', action='store_true', help='X HELP'), Sig('--y', help='Y HELP'), Sig('ekiekiekifekang', help='EKI HELP'), Sig('bar', help='BAR HELP'), ] argument_group_signatures = [] usage = '''\ usage: USAGE ''' help = usage + '''\ DESCRIPTION positional arguments: ekiekiekifekang EKI HELP bar BAR HELP optional arguments: -h, --help show this help message and exit -x X HELP --y Y Y HELP ''' version = '' class TestHelpReformatting(HelpTestCase): """Make sure that text after short names starts on the first line""" parser_signature = Sig( prog='PROG', description=' oddly formatted\n' 'description\n' '\n' 'that is so long that it should go onto multiple ' 'lines when wrapped') argument_signatures = [ Sig('-x', metavar='XX', help='oddly\n' ' formatted -x help'), Sig('y', metavar='yyy', help='normal y help'), ] argument_group_signatures = [ (Sig('title', description='\n' ' oddly formatted group\n' '\n' 'description'), [Sig('-a', action='store_true', help=' oddly \n' 'formatted -a help \n' ' again, so long that it should be wrapped over ' 'multiple lines')]), ] usage = '''\ usage: PROG [-h] [-x XX] [-a] yyy ''' help = usage + '''\ oddly formatted description that is so long that it should go onto \ multiple lines when wrapped positional arguments: yyy normal y help optional arguments: -h, --help show this help message and exit -x XX oddly formatted -x help title: oddly formatted group description -a oddly formatted -a help again, so long that it should \ be wrapped over multiple lines ''' version = '' class TestHelpWrappingShortNames(HelpTestCase): """Make sure that text after short names starts on the first line""" parser_signature = Sig(prog='PROG', description= 'D\nD' * 30) argument_signatures = [ Sig('-x', metavar='XX', help='XHH HX' * 20), Sig('y', metavar='yyy', help='YH YH' * 20), ] argument_group_signatures = [ (Sig('ALPHAS'), [ Sig('-a', action='store_true', help='AHHH HHA' * 10)]), ] usage = '''\ usage: PROG [-h] [-x XX] [-a] yyy ''' help = usage + '''\ D DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD \ DD DD DD DD DD DD DD D positional arguments: yyy YH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH \ YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YH optional arguments: -h, --help show this help message and exit -x XX XHH HXXHH HXXHH HXXHH HXXHH HXXHH HXXHH HXXHH HXXHH \ HXXHH HXXHH HXXHH HXXHH HXXHH HXXHH HXXHH HXXHH HXXHH HXXHH HXXHH HX ALPHAS: -a AHHH HHAAHHH HHAAHHH HHAAHHH HHAAHHH HHAAHHH HHAAHHH \ HHAAHHH HHAAHHH HHAAHHH HHA ''' version = '' class TestHelpWrappingLongNames(HelpTestCase): """Make sure that text after long names starts on the next line""" parser_signature = Sig(usage='USAGE', description= 'D D' * 30, version='V V'*30) argument_signatures = [ Sig('-x', metavar='X' * 25, help='XH XH' * 20), Sig('y', metavar='y' * 25, help='YH YH' * 20), ] argument_group_signatures = [ (Sig('ALPHAS'), [ Sig('-a', metavar='A' * 25, help='AH AH' * 20), Sig('z', metavar='z' * 25, help='ZH ZH' * 20)]), ] usage = '''\ usage: USAGE ''' help = usage + '''\ D DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD DD \ DD DD DD DD DD DD DD D positional arguments: yyyyyyyyyyyyyyyyyyyyyyyyy YH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH \ YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YHYH YH optional arguments: -h, --help show this help message and exit -v, --version show program's version number and exit -x XXXXXXXXXXXXXXXXXXXXXXXXX XH XHXH XHXH XHXH XHXH XHXH XHXH XHXH XHXH \ XHXH XHXH XHXH XHXH XHXH XHXH XHXH XHXH XHXH XHXH XHXH XH ALPHAS: -a AAAAAAAAAAAAAAAAAAAAAAAAA AH AHAH AHAH AHAH AHAH AHAH AHAH AHAH AHAH \ AHAH AHAH AHAH AHAH AHAH AHAH AHAH AHAH AHAH AHAH AHAH AH zzzzzzzzzzzzzzzzzzzzzzzzz ZH ZHZH ZHZH ZHZH ZHZH ZHZH ZHZH ZHZH ZHZH \ ZHZH ZHZH ZHZH ZHZH ZHZH ZHZH ZHZH ZHZH ZHZH ZHZH ZHZH ZH ''' version = '''\ V VV VV VV VV VV VV VV VV VV VV VV VV VV VV VV VV VV VV VV VV VV VV \ VV VV VV VV VV VV VV V ''' class TestHelpUsage(HelpTestCase): """Test basic usage messages""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('-w', nargs='+', help='w'), Sig('-x', nargs='*', help='x'), Sig('a', help='a'), Sig('b', help='b', nargs=2), Sig('c', help='c', nargs='?'), ] argument_group_signatures = [ (Sig('group'), [ Sig('-y', nargs='?', help='y'), Sig('-z', nargs=3, help='z'), Sig('d', help='d', nargs='*'), Sig('e', help='e', nargs='+'), ]) ] usage = '''\ usage: PROG [-h] [-w W [W ...]] [-x [X [X ...]]] [-y [Y]] [-z Z Z Z] a b b [c] [d [d ...]] e [e ...] ''' help = usage + '''\ positional arguments: a a b b c c optional arguments: -h, --help show this help message and exit -w W [W ...] w -x [X [X ...]] x group: -y [Y] y -z Z Z Z z d d e e ''' version = '' class TestHelpOnlyUserGroups(HelpTestCase): """Test basic usage messages""" parser_signature = Sig(prog='PROG', add_help=False) argument_signatures = [] argument_group_signatures = [ (Sig('xxxx'), [ Sig('-x', help='x'), Sig('a', help='a'), ]), (Sig('yyyy'), [ Sig('b', help='b'), Sig('-y', help='y'), ]), ] usage = '''\ usage: PROG [-x X] [-y Y] a b ''' help = usage + '''\ xxxx: -x X x a a yyyy: b b -y Y y ''' version = '' class TestHelpUsageLongProg(HelpTestCase): """Test usage messages where the prog is long""" parser_signature = Sig(prog='P' * 60) argument_signatures = [ Sig('-w', metavar='W'), Sig('-x', metavar='X'), Sig('a'), Sig('b'), ] argument_group_signatures = [] usage = '''\ usage: PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP [-h] [-w W] [-x X] a b ''' help = usage + '''\ positional arguments: a b optional arguments: -h, --help show this help message and exit -w W -x X ''' version = '' class TestHelpUsageLongProgOptionsWrap(HelpTestCase): """Test usage messages where the prog is long and the optionals wrap""" parser_signature = Sig(prog='P' * 60) argument_signatures = [ Sig('-w', metavar='W' * 25), Sig('-x', metavar='X' * 25), Sig('-y', metavar='Y' * 25), Sig('-z', metavar='Z' * 25), Sig('a'), Sig('b'), ] argument_group_signatures = [] usage = '''\ usage: PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP [-h] [-w WWWWWWWWWWWWWWWWWWWWWWWWW] \ [-x XXXXXXXXXXXXXXXXXXXXXXXXX] [-y YYYYYYYYYYYYYYYYYYYYYYYYY] [-z ZZZZZZZZZZZZZZZZZZZZZZZZZ] a b ''' help = usage + '''\ positional arguments: a b optional arguments: -h, --help show this help message and exit -w WWWWWWWWWWWWWWWWWWWWWWWWW -x XXXXXXXXXXXXXXXXXXXXXXXXX -y YYYYYYYYYYYYYYYYYYYYYYYYY -z ZZZZZZZZZZZZZZZZZZZZZZZZZ ''' version = '' class TestHelpUsageLongProgPositionalsWrap(HelpTestCase): """Test usage messages where the prog is long and the positionals wrap""" parser_signature = Sig(prog='P' * 60, add_help=False) argument_signatures = [ Sig('a' * 25), Sig('b' * 25), Sig('c' * 25), ] argument_group_signatures = [] usage = '''\ usage: PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP aaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbb ccccccccccccccccccccccccc ''' help = usage + '''\ positional arguments: aaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbb ccccccccccccccccccccccccc ''' version = '' class TestHelpUsageOptionalsWrap(HelpTestCase): """Test usage messages where the optionals wrap""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('-w', metavar='W' * 25), Sig('-x', metavar='X' * 25), Sig('-y', metavar='Y' * 25), Sig('-z', metavar='Z' * 25), Sig('a'), Sig('b'), Sig('c'), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [-w WWWWWWWWWWWWWWWWWWWWWWWWW] \ [-x XXXXXXXXXXXXXXXXXXXXXXXXX] [-y YYYYYYYYYYYYYYYYYYYYYYYYY] \ [-z ZZZZZZZZZZZZZZZZZZZZZZZZZ] a b c ''' help = usage + '''\ positional arguments: a b c optional arguments: -h, --help show this help message and exit -w WWWWWWWWWWWWWWWWWWWWWWWWW -x XXXXXXXXXXXXXXXXXXXXXXXXX -y YYYYYYYYYYYYYYYYYYYYYYYYY -z ZZZZZZZZZZZZZZZZZZZZZZZZZ ''' version = '' class TestHelpUsagePositionalsWrap(HelpTestCase): """Test usage messages where the positionals wrap""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('-x'), Sig('-y'), Sig('-z'), Sig('a' * 25), Sig('b' * 25), Sig('c' * 25), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [-x X] [-y Y] [-z Z] aaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbb ccccccccccccccccccccccccc ''' help = usage + '''\ positional arguments: aaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbb ccccccccccccccccccccccccc optional arguments: -h, --help show this help message and exit -x X -y Y -z Z ''' version = '' class TestHelpUsageOptionalsPositionalsWrap(HelpTestCase): """Test usage messages where the optionals and positionals wrap""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('-x', metavar='X' * 25), Sig('-y', metavar='Y' * 25), Sig('-z', metavar='Z' * 25), Sig('a' * 25), Sig('b' * 25), Sig('c' * 25), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [-x XXXXXXXXXXXXXXXXXXXXXXXXX] \ [-y YYYYYYYYYYYYYYYYYYYYYYYYY] [-z ZZZZZZZZZZZZZZZZZZZZZZZZZ] aaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbb ccccccccccccccccccccccccc ''' help = usage + '''\ positional arguments: aaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbb ccccccccccccccccccccccccc optional arguments: -h, --help show this help message and exit -x XXXXXXXXXXXXXXXXXXXXXXXXX -y YYYYYYYYYYYYYYYYYYYYYYYYY -z ZZZZZZZZZZZZZZZZZZZZZZZZZ ''' version = '' class TestHelpUsageOptionalsOnlyWrap(HelpTestCase): """Test usage messages where there are only optionals and they wrap""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('-x', metavar='X' * 25), Sig('-y', metavar='Y' * 25), Sig('-z', metavar='Z' * 25), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [-x XXXXXXXXXXXXXXXXXXXXXXXXX] \ [-y YYYYYYYYYYYYYYYYYYYYYYYYY] [-z ZZZZZZZZZZZZZZZZZZZZZZZZZ] ''' help = usage + '''\ optional arguments: -h, --help show this help message and exit -x XXXXXXXXXXXXXXXXXXXXXXXXX -y YYYYYYYYYYYYYYYYYYYYYYYYY -z ZZZZZZZZZZZZZZZZZZZZZZZZZ ''' version = '' class TestHelpUsagePositionalsOnlyWrap(HelpTestCase): """Test usage messages where there are only positionals and they wrap""" parser_signature = Sig(prog='PROG', add_help=False) argument_signatures = [ Sig('a' * 25), Sig('b' * 25), Sig('c' * 25), ] argument_group_signatures = [] usage = '''\ usage: PROG aaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbb ccccccccccccccccccccccccc ''' help = usage + '''\ positional arguments: aaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbbbbbbb ccccccccccccccccccccccccc ''' version = '' class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('-x', type=int, help='x %(prog)s %(default)s %(type)s %%'), Sig('-y', action='store_const', default=42, const='XXX', help='y %(prog)s %(default)s %(const)s'), Sig('--foo', choices='abc', help='foo %(prog)s %(default)s %(choices)s'), Sig('--bar', default='baz', choices=[1, 2], metavar='BBB', help='bar %(prog)s %(default)s %(dest)s'), Sig('spam', help='spam %(prog)s %(default)s'), Sig('badger', default=0.5, help='badger %(prog)s %(default)s'), ] argument_group_signatures = [ (Sig('group'), [ Sig('-a', help='a %(prog)s %(default)s'), Sig('-b', default=-1, help='b %(prog)s %(default)s'), ]) ] usage = ('''\ usage: PROG [-h] [-x X] [-y] [--foo {a,b,c}] [--bar BBB] [-a A] [-b B] spam badger ''') help = usage + '''\ positional arguments: spam spam PROG None badger badger PROG 0.5 optional arguments: -h, --help show this help message and exit -x X x PROG None int % -y y PROG 42 XXX --foo {a,b,c} foo PROG None a, b, c --bar BBB bar PROG baz bar group: -a A a PROG None -b B b PROG -1 ''' version = '' class TestHelpVariableExpansionUsageSupplied(HelpTestCase): """Test that variables are expanded properly when usage= is present""" parser_signature = Sig(prog='PROG', usage='%(prog)s FOO') argument_signatures = [] argument_group_signatures = [] usage = ('''\ usage: PROG FOO ''') help = usage + '''\ optional arguments: -h, --help show this help message and exit ''' version = '' class TestHelpVariableExpansionNoArguments(HelpTestCase): """Test that variables are expanded properly with no arguments""" parser_signature = Sig(prog='PROG', add_help=False) argument_signatures = [] argument_group_signatures = [] usage = ('''\ usage: PROG ''') help = usage version = '' class TestHelpSuppressUsage(HelpTestCase): """Test that items can be suppressed in usage messages""" parser_signature = Sig(prog='PROG', usage=argparse.SUPPRESS) argument_signatures = [ Sig('--foo', help='foo help'), Sig('spam', help='spam help'), ] argument_group_signatures = [] help = '''\ positional arguments: spam spam help optional arguments: -h, --help show this help message and exit --foo FOO foo help ''' usage = '' version = '' class TestHelpSuppressOptional(HelpTestCase): """Test that optional arguments can be suppressed in help messages""" parser_signature = Sig(prog='PROG', add_help=False) argument_signatures = [ Sig('--foo', help=argparse.SUPPRESS), Sig('spam', help='spam help'), ] argument_group_signatures = [] usage = '''\ usage: PROG spam ''' help = usage + '''\ positional arguments: spam spam help ''' version = '' class TestHelpSuppressOptionalGroup(HelpTestCase): """Test that optional groups can be suppressed in help messages""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('--foo', help='foo help'), Sig('spam', help='spam help'), ] argument_group_signatures = [ (Sig('group'), [Sig('--bar', help=argparse.SUPPRESS)]), ] usage = '''\ usage: PROG [-h] [--foo FOO] spam ''' help = usage + '''\ positional arguments: spam spam help optional arguments: -h, --help show this help message and exit --foo FOO foo help ''' version = '' class TestHelpSuppressPositional(HelpTestCase): """Test that positional arguments can be suppressed in help messages""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('--foo', help='foo help'), Sig('spam', help=argparse.SUPPRESS), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [--foo FOO] ''' help = usage + '''\ optional arguments: -h, --help show this help message and exit --foo FOO foo help ''' version = '' class TestHelpRequiredOptional(HelpTestCase): """Test that required options don't look optional""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('--foo', required=True, help='foo help'), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] --foo FOO ''' help = usage + '''\ optional arguments: -h, --help show this help message and exit --foo FOO foo help ''' version = '' class TestHelpAlternatePrefixChars(HelpTestCase): """Test that options display with different prefix characters""" parser_signature = Sig(prog='PROG', prefix_chars='^;', add_help=False) argument_signatures = [ Sig('^^foo', action='store_true', help='foo help'), Sig(';b', ';;bar', help='bar help'), ] argument_group_signatures = [] usage = '''\ usage: PROG [^^foo] [;b BAR] ''' help = usage + '''\ optional arguments: ^^foo foo help ;b BAR, ;;bar BAR bar help ''' version = '' class TestHelpNoHelpOptional(HelpTestCase): """Test that the --help argument can be suppressed help messages""" parser_signature = Sig(prog='PROG', add_help=False) argument_signatures = [ Sig('--foo', help='foo help'), Sig('spam', help='spam help'), ] argument_group_signatures = [] usage = '''\ usage: PROG [--foo FOO] spam ''' help = usage + '''\ positional arguments: spam spam help optional arguments: --foo FOO foo help ''' version = '' class TestHelpVersionOptional(HelpTestCase): """Test that the --version argument can be suppressed help messages""" parser_signature = Sig(prog='PROG', version='1.0') argument_signatures = [ Sig('--foo', help='foo help'), Sig('spam', help='spam help'), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [-v] [--foo FOO] spam ''' help = usage + '''\ positional arguments: spam spam help optional arguments: -h, --help show this help message and exit -v, --version show program's version number and exit --foo FOO foo help ''' version = '''\ 1.0 ''' class TestHelpNone(HelpTestCase): """Test that no errors occur if no help is specified""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('--foo'), Sig('spam'), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [--foo FOO] spam ''' help = usage + '''\ positional arguments: spam optional arguments: -h, --help show this help message and exit --foo FOO ''' version = '' class TestHelpTupleMetavar(HelpTestCase): """Test specifying metavar as a tuple""" parser_signature = Sig(prog='PROG') argument_signatures = [ Sig('-w', help='w', nargs='+', metavar=('W1', 'W2')), Sig('-x', help='x', nargs='*', metavar=('X1', 'X2')), Sig('-y', help='y', nargs=3, metavar=('Y1', 'Y2', 'Y3')), Sig('-z', help='z', nargs='?', metavar=('Z1', )), ] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [-w W1 [W2 ...]] [-x [X1 [X2 ...]]] [-y Y1 Y2 Y3] \ [-z [Z1]] ''' help = usage + '''\ optional arguments: -h, --help show this help message and exit -w W1 [W2 ...] w -x [X1 [X2 ...]] x -y Y1 Y2 Y3 y -z [Z1] z ''' version = '' class TestHelpRawText(HelpTestCase): """Test the RawTextHelpFormatter""" parser_signature = Sig( prog='PROG', formatter_class=argparse.RawTextHelpFormatter, description='Keep the formatting\n' ' exactly as it is written\n' '\n' 'here\n') argument_signatures = [ Sig('--foo', help=' foo help should also\n' 'appear as given here'), Sig('spam', help='spam help'), ] argument_group_signatures = [ (Sig('title', description=' This text\n' ' should be indented\n' ' exactly like it is here\n'), [Sig('--bar', help='bar help')]), ] usage = '''\ usage: PROG [-h] [--foo FOO] [--bar BAR] spam ''' help = usage + '''\ Keep the formatting exactly as it is written here positional arguments: spam spam help optional arguments: -h, --help show this help message and exit --foo FOO foo help should also appear as given here title: This text should be indented exactly like it is here --bar BAR bar help ''' version = '' class TestHelpRawDescription(HelpTestCase): """Test the RawTextHelpFormatter""" parser_signature = Sig( prog='PROG', formatter_class=argparse.RawDescriptionHelpFormatter, description='Keep the formatting\n' ' exactly as it is written\n' '\n' 'here\n') argument_signatures = [ Sig('--foo', help=' foo help should not\n' ' retain this odd formatting'), Sig('spam', help='spam help'), ] argument_group_signatures = [ (Sig('title', description=' This text\n' ' should be indented\n' ' exactly like it is here\n'), [Sig('--bar', help='bar help')]), ] usage = '''\ usage: PROG [-h] [--foo FOO] [--bar BAR] spam ''' help = usage + '''\ Keep the formatting exactly as it is written here positional arguments: spam spam help optional arguments: -h, --help show this help message and exit --foo FOO foo help should not retain this odd formatting title: This text should be indented exactly like it is here --bar BAR bar help ''' version = '' class TestHelpArgumentDefaults(HelpTestCase): """Test the ArgumentDefaultsHelpFormatter""" parser_signature = Sig( prog='PROG', formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='description') argument_signatures = [ Sig('--foo', help='foo help - oh and by the way, %(default)s'), Sig('--bar', action='store_true', help='bar help'), Sig('spam', help='spam help'), Sig('badger', nargs='?', default='wooden', help='badger help'), ] argument_group_signatures = [ (Sig('title', description='description'), [Sig('--baz', type=int, default=42, help='baz help')]), ] usage = '''\ usage: PROG [-h] [--foo FOO] [--bar] [--baz BAZ] spam [badger] ''' help = usage + '''\ description positional arguments: spam spam help badger badger help (default: wooden) optional arguments: -h, --help show this help message and exit --foo FOO foo help - oh and by the way, None --bar bar help (default: False) title: description --baz BAZ baz help (default: 42) ''' version = '' class TestHelpVersionAction(HelpTestCase): """Test the default help for the version action""" parser_signature = Sig(prog='PROG', description='description') argument_signatures = [Sig('-V', '--version', action='version', version='3.6')] argument_group_signatures = [] usage = '''\ usage: PROG [-h] [-V] ''' help = usage + '''\ description optional arguments: -h, --help show this help message and exit -V, --version show program's version number and exit ''' version = '' class TestHelpSubparsersOrdering(HelpTestCase): """Test ordering of subcommands in help matches the code""" parser_signature = Sig(prog='PROG', description='display some subcommands', version='0.1') subparsers_signatures = [Sig(name=name) for name in ('a', 'b', 'c', 'd', 'e')] usage = '''\ usage: PROG [-h] [-v] {a,b,c,d,e} ... ''' help = usage + '''\ display some subcommands positional arguments: {a,b,c,d,e} optional arguments: -h, --help show this help message and exit -v, --version show program's version number and exit ''' version = '''\ 0.1 ''' # class TestHelpSubparsersWithHelpOrdering(HelpTestCase): # """Test ordering of subcommands in help matches the code""" # parser_signature = Sig(prog='PROG', # description='display some subcommands', # version='0.1') # subcommand_data = (('a', 'a subcommand help'), # ('b', 'b subcommand help'), # ('c', 'c subcommand help'), # ('d', 'd subcommand help'), # ('e', 'e subcommand help'), # ) # # TODO: Allow accessing class vars inside class expr (e.g. subcommand_data) # subparsers_signatures = [Sig(name=name, help=help) # for name, help in subcommand_data] # usage = '''\ # usage: PROG [-h] [-v] {a,b,c,d,e} ... # ''' # help = usage + '''\ # display some subcommands # positional arguments: # {a,b,c,d,e} # a a subcommand help # b b subcommand help # c c subcommand help # d d subcommand help # e e subcommand help # optional arguments: # -h, --help show this help message and exit # -v, --version show program's version number and exit # ''' # version = '''\ # 0.1 # ''' # ===================================== # Optional/Positional constructor tests # ===================================== class TestInvalidArgumentConstructors(TestCase): """Test a bunch of invalid Argument constructors""" def assertTypeError(self, *args, **kwargs): parser = argparse.ArgumentParser() self.assertRaises(TypeError, parser.add_argument, *args, **kwargs) def assertValueError(self, *args, **kwargs): parser = argparse.ArgumentParser() self.assertRaises(ValueError, parser.add_argument, *args, **kwargs) def test_invalid_keyword_arguments(self): self.assertTypeError('-x', bar=None) self.assertTypeError('-y', callback='foo') self.assertTypeError('-y', callback_args=()) self.assertTypeError('-y', callback_kwargs={}) def test_missing_destination(self): self.assertTypeError() for action in ['append', 'store']: self.assertTypeError(action=action) def test_invalid_option_strings(self): self.assertValueError('--') self.assertValueError('---') def test_invalid_type(self): self.assertValueError('--foo', type='int') self.assertValueError('--foo', type=(int, float)) def test_invalid_action(self): self.assertValueError('-x', action='foo') self.assertValueError('foo', action='baz') self.assertValueError('--foo', action=('store', 'append')) parser = argparse.ArgumentParser() try: parser.add_argument("--foo", action="store-true") except ValueError: e = sys.exc_info()[1] expected = 'unknown action' msg = 'expected %r, found %r' % (expected, e) self.assertTrue(expected in str(e), msg) def test_multiple_dest(self): parser = argparse.ArgumentParser() parser.add_argument(dest='foo') try: parser.add_argument('bar', dest='baz') except ValueError: e = sys.exc_info()[1] expected = 'dest supplied twice for positional argument' msg = 'expected %r, found %r' % (expected, e) self.assertTrue(expected in str(e), msg) def test_no_argument_actions(self): for action in ['store_const', 'store_true', 'store_false', 'append_const', 'count']: for attrs in [dict(type=int), dict(nargs='+'), dict(choices='ab')]: self.assertTypeError('-x', action=action, **attrs) def test_no_argument_no_const_actions(self): # options with zero arguments for action in ['store_true', 'store_false', 'count']: # const is always disallowed self.assertTypeError('-x', const='foo', action=action) # nargs is always disallowed self.assertTypeError('-x', nargs='*', action=action) def test_more_than_one_argument_actions(self): for action in ['store', 'append']: # nargs=0 is disallowed self.assertValueError('-x', nargs=0, action=action) self.assertValueError('spam', nargs=0, action=action) # const is disallowed with non-optional arguments for nargs in [1, '*', '+']: self.assertValueError('-x', const='foo', nargs=nargs, action=action) self.assertValueError('spam', const='foo', nargs=nargs, action=action) def test_required_const_actions(self): for action in ['store_const', 'append_const']: # nargs is always disallowed self.assertTypeError('-x', nargs='+', action=action) def test_parsers_action_missing_params(self): self.assertTypeError('command', action='parsers') self.assertTypeError('command', action='parsers', prog='PROG') self.assertTypeError('command', action='parsers', parser_class=argparse.ArgumentParser) def test_required_positional(self): self.assertTypeError('foo', required=True) def test_user_defined_action(self): class Success(Exception): pass class Action(object): def __init__(self, option_strings, dest, const, default, required=False): if dest == 'spam': if const is Success: if default is Success: raise Success() def __call__(self, *args, **kwargs): pass parser = argparse.ArgumentParser() self.assertRaises(Success, parser.add_argument, '--spam', action=Action, default=Success, const=Success) self.assertRaises(Success, parser.add_argument, 'spam', action=Action, default=Success, const=Success) # ================================ # Actions returned by add_argument # ================================ class TestActionsReturned(TestCase): def test_dest(self): parser = argparse.ArgumentParser() action = parser.add_argument('--foo') self.assertEqual(action.dest, 'foo') action = parser.add_argument('-b', '--bar') self.assertEqual(action.dest, 'bar') action = parser.add_argument('-x', '-y') self.assertEqual(action.dest, 'x') def test_misc(self): parser = argparse.ArgumentParser() action = parser.add_argument('--foo', nargs='?', const=42, default=84, type=int, choices=[1, 2], help='FOO', metavar='BAR', dest='baz') self.assertEqual(action.nargs, '?') self.assertEqual(action.const, 42) self.assertEqual(action.default, 84) self.assertEqual(action.type, int) self.assertEqual(action.choices, [1, 2]) self.assertEqual(action.help, 'FOO') self.assertEqual(action.metavar, 'BAR') self.assertEqual(action.dest, 'baz') # ================================ # Argument conflict handling tests # ================================ class TestConflictHandling(TestCase): def test_bad_type(self): self.assertRaises(ValueError, argparse.ArgumentParser, conflict_handler='foo') def test_conflict_error(self): parser = argparse.ArgumentParser() parser.add_argument('-x') self.assertRaises(argparse.ArgumentError, parser.add_argument, '-x') parser.add_argument('--spam') self.assertRaises(argparse.ArgumentError, parser.add_argument, '--spam') def test_resolve_error(self): get_parser = argparse.ArgumentParser parser = get_parser(prog='PROG', conflict_handler='resolve') parser.add_argument('-x', help='OLD X') parser.add_argument('-x', help='NEW X') self.assertEqual(parser.format_help(), textwrap.dedent('''\ usage: PROG [-h] [-x X] optional arguments: -h, --help show this help message and exit -x X NEW X ''')) parser.add_argument('--spam', metavar='OLD_SPAM') parser.add_argument('--spam', metavar='NEW_SPAM') self.assertEqual(parser.format_help(), textwrap.dedent('''\ usage: PROG [-h] [-x X] [--spam NEW_SPAM] optional arguments: -h, --help show this help message and exit -x X NEW X --spam NEW_SPAM ''')) # ============================= # Help and Version option tests # ============================= class TestOptionalsHelpVersionActions(TestCase): """Test the help and version actions""" def _get_error(self, func, *args, **kwargs): try: func(*args, **kwargs) except ArgumentParserError: return sys.exc_info()[1] else: self.assertRaises(ArgumentParserError, func, *args, **kwargs) def assertPrintHelpExit(self, parser, args_str): self.assertEqual( parser.format_help(), self._get_error(parser.parse_args, args_str.split()).stdout) def assertPrintVersionExit(self, parser, args_str): self.assertEqual( parser.format_version(), self._get_error(parser.parse_args, args_str.split()).stderr) def assertArgumentParserError(self, parser, *args): self.assertRaises(ArgumentParserError, parser.parse_args, args) def test_version(self): parser = ErrorRaisingArgumentParser(version='1.0') self.assertPrintHelpExit(parser, '-h') self.assertPrintHelpExit(parser, '--help') self.assertPrintVersionExit(parser, '-v') self.assertPrintVersionExit(parser, '--version') def test_version_format(self): parser = ErrorRaisingArgumentParser(prog='PPP', version='%(prog)s 3.5') msg = self._get_error(parser.parse_args, ['-v']).stderr self.assertEqual('PPP 3.5\n', msg) def test_version_no_help(self): parser = ErrorRaisingArgumentParser(add_help=False, version='1.0') self.assertArgumentParserError(parser, '-h') self.assertArgumentParserError(parser, '--help') self.assertPrintVersionExit(parser, '-v') self.assertPrintVersionExit(parser, '--version') def test_version_action(self): parser = ErrorRaisingArgumentParser(prog='XXX') parser.add_argument('-V', action='version', version='%(prog)s 3.7') msg = self._get_error(parser.parse_args, ['-V']).stderr self.assertEqual('XXX 3.7\n', msg) def test_no_help(self): parser = ErrorRaisingArgumentParser(add_help=False) self.assertArgumentParserError(parser, '-h') self.assertArgumentParserError(parser, '--help') self.assertArgumentParserError(parser, '-v') self.assertArgumentParserError(parser, '--version') def test_alternate_help_version(self): parser = ErrorRaisingArgumentParser() parser.add_argument('-x', action='help') parser.add_argument('-y', action='version') self.assertPrintHelpExit(parser, '-x') self.assertPrintVersionExit(parser, '-y') self.assertArgumentParserError(parser, '-v') self.assertArgumentParserError(parser, '--version') def test_help_version_extra_arguments(self): parser = ErrorRaisingArgumentParser(version='1.0') parser.add_argument('-x', action='store_true') parser.add_argument('y') # try all combinations of valid prefixes and suffixes valid_prefixes = ['', '-x', 'foo', '-x bar', 'baz -x'] valid_suffixes = valid_prefixes + ['--bad-option', 'foo bar baz'] for prefix in valid_prefixes: for suffix in valid_suffixes: format = '%s %%s %s' % (prefix, suffix) self.assertPrintHelpExit(parser, format % '-h') self.assertPrintHelpExit(parser, format % '--help') self.assertPrintVersionExit(parser, format % '-v') self.assertPrintVersionExit(parser, format % '--version') # ====================== # str() and repr() tests # ====================== class TestStrings(TestCase): """Test str() and repr() on Optionals and Positionals""" def assertStringEqual(self, obj, result_string): for func in [str, repr]: self.assertEqual(func(obj), result_string) def test_optional(self): option = argparse.Action( option_strings=['--foo', '-a', '-b'], dest='b', type='int', nargs='+', default=42, choices=[1, 2, 3], help='HELP', metavar='METAVAR') string = ( "Action(option_strings=['--foo', '-a', '-b'], dest='b', " "nargs='+', const=None, default=42, type='int', " "choices=[1, 2, 3], help='HELP', metavar='METAVAR')") self.assertStringEqual(option, string) def test_argument(self): argument = argparse.Action( option_strings=[], dest='x', type=float, nargs='?', default=2.5, choices=[0.5, 1.5, 2.5], help='H HH H', metavar='MV MV MV') string = ( "Action(option_strings=[], dest='x', nargs='?', " "const=None, default=2.5, type=%r, choices=[0.5, 1.5, 2.5], " "help='H HH H', metavar='MV MV MV')" % float) self.assertStringEqual(argument, string) def test_namespace(self): ns = argparse.Namespace(foo=42, bar='spam') string = "Namespace(bar='spam', foo=42)" self.assertStringEqual(ns, string) def test_parser(self): parser = argparse.ArgumentParser(prog='PROG') string = ( "ArgumentParser(prog='PROG', usage=None, description=None, " "version=None, formatter_class=%r, conflict_handler='error', " "add_help=True)" % argparse.HelpFormatter) self.assertStringEqual(parser, string) # =============== # Namespace tests # =============== class TestNamespace(TestCase): def test_constructor(self): ns = argparse.Namespace() self.assertRaises(AttributeError, getattr, ns, 'x') ns = argparse.Namespace(a=42, b='spam') self.assertEqual(ns.a, 42) self.assertEqual(ns.b, 'spam') def test_equality(self): ns1 = argparse.Namespace(a=1, b=2) ns2 = argparse.Namespace(b=2, a=1) ns3 = argparse.Namespace(a=1) ns4 = argparse.Namespace(b=2) self.assertEqual(ns1, ns2) self.assertNotEqual(ns1, ns3) self.assertNotEqual(ns1, ns4) self.assertNotEqual(ns2, ns3) self.assertNotEqual(ns2, ns4) self.assertTrue(ns1 != ns3) self.assertTrue(ns1 != ns4) self.assertTrue(ns2 != ns3) self.assertTrue(ns2 != ns4) def test_equality_returns_notimplemeted(self): # See issue 21481 ns = argparse.Namespace(a=1, b=2) self.assertIs(ns.__eq__(None), NotImplemented) self.assertIs(ns.__ne__(None), NotImplemented) # =================== # File encoding tests # =================== # class TestEncoding(TestCase): # def _test_module_encoding(self, path): # path, _ = os.path.splitext(path) # path += ".py" # with codecs.open(path, 'r', 'utf8') as f: # f.read() # def test_argparse_module_encoding(self): # self._test_module_encoding(argparse.__file__) # def test_test_argparse_module_encoding(self): # self._test_module_encoding(__file__) # =================== # ArgumentError tests # =================== class TestArgumentError(TestCase): def test_argument_error(self): msg = "my error here" error = argparse.ArgumentError(None, msg) self.assertEqual(str(error), msg) # ======================= # ArgumentTypeError tests # ======================= class TestArgumentTypeError(TestCase): def test_argument_type_error(self): def spam(string): raise argparse.ArgumentTypeError('spam!') parser = ErrorRaisingArgumentParser(prog='PROG', add_help=False) parser.add_argument('x', type=spam) try: parser.parse_args(['XXX']) except ArgumentParserError: expected = 'usage: PROG x\nPROG: error: argument x: spam!\n' msg = sys.exc_info()[1].stderr self.assertEqual(expected, msg) else: self.fail() # ================================================ # Check that the type function is called only once # ================================================ class TestTypeFunctionCallOnlyOnce(TestCase): def test_type_function_call_only_once(self): def spam(string_to_convert): self.assertEqual(string_to_convert, 'spam!') return 'foo_converted' parser = argparse.ArgumentParser() parser.add_argument('--foo', type=spam, default='bar') args = parser.parse_args('--foo spam!'.split()) self.assertEqual(NS(foo='foo_converted'), args) # ================================================================== # Check semantics regarding the default argument and type conversion # ================================================================== class TestTypeFunctionCalledOnDefault(TestCase): def test_type_function_call_with_non_string_default(self): def spam(int_to_convert): self.assertEqual(int_to_convert, 0) return 'foo_converted' parser = argparse.ArgumentParser() parser.add_argument('--foo', type=spam, default=0) args = parser.parse_args([]) # foo should *not* be converted because its default is not a string. self.assertEqual(NS(foo=0), args) def test_type_function_call_with_string_default(self): def spam(int_to_convert): return 'foo_converted' parser = argparse.ArgumentParser() parser.add_argument('--foo', type=spam, default='0') args = parser.parse_args([]) # foo is converted because its default is a string. self.assertEqual(NS(foo='foo_converted'), args) def test_no_double_type_conversion_of_default(self): def extend(str_to_convert): return str_to_convert + '*' parser = argparse.ArgumentParser() parser.add_argument('--test', type=extend, default='*') args = parser.parse_args([]) # The test argument will be two stars, one coming from the default # value and one coming from the type conversion being called exactly # once. self.assertEqual(NS(test='**'), args) def test_issue_15906(self): # Issue #15906: When action='append', type=str, default=[] are # providing, the dest value was the string representation "[]" when it # should have been an empty list. parser = argparse.ArgumentParser() parser.add_argument('--test', dest='test', type=str, default=[], action='append') args = parser.parse_args([]) self.assertEqual(args.test, []) # ====================== # parse_known_args tests # ====================== class TestParseKnownArgs(TestCase): def test_arguments_tuple(self): parser = argparse.ArgumentParser() parser.parse_args(()) def test_arguments_list(self): parser = argparse.ArgumentParser() parser.parse_args([]) def test_arguments_tuple_positional(self): parser = argparse.ArgumentParser() parser.add_argument('x') parser.parse_args(('x',)) def test_arguments_list_positional(self): parser = argparse.ArgumentParser() parser.add_argument('x') parser.parse_args(['x']) def test_optionals(self): parser = argparse.ArgumentParser() parser.add_argument('--foo') args, extras = parser.parse_known_args('--foo F --bar --baz'.split()) self.assertEqual(NS(foo='F'), args) self.assertEqual(['--bar', '--baz'], extras) def test_mixed(self): parser = argparse.ArgumentParser() parser.add_argument('-v', nargs='?', const=1, type=int) parser.add_argument('--spam', action='store_false') parser.add_argument('badger') argv = ["B", "C", "--foo", "-v", "3", "4"] args, extras = parser.parse_known_args(argv) self.assertEqual(NS(v=3, spam=True, badger="B"), args) self.assertEqual(["C", "--foo", "4"], extras) # ========================== # add_argument metavar tests # ========================== class TestAddArgumentMetavar(TestCase): EXPECTED_MESSAGE = "length of metavar tuple does not match nargs" def do_test_no_exception(self, nargs, metavar): parser = argparse.ArgumentParser() parser.add_argument("--foo", nargs=nargs, metavar=metavar) def do_test_exception(self, nargs, metavar): parser = argparse.ArgumentParser() with self.assertRaises(ValueError) as cm: parser.add_argument("--foo", nargs=nargs, metavar=metavar) # self.assertEqual(cm.exception.args[0], self.EXPECTED_MESSAGE) self.assertEqual(str(cm.exception), self.EXPECTED_MESSAGE) # Unit tests for different values of metavar when nargs=None def test_nargs_None_metavar_string(self): self.do_test_no_exception(nargs=None, metavar="1") def test_nargs_None_metavar_length0(self): self.do_test_exception(nargs=None, metavar=tuple()) def test_nargs_None_metavar_length1(self): self.do_test_no_exception(nargs=None, metavar=("1")) def test_nargs_None_metavar_length2(self): self.do_test_exception(nargs=None, metavar=("1", "2")) def test_nargs_None_metavar_length3(self): self.do_test_exception(nargs=None, metavar=("1", "2", "3")) # Unit tests for different values of metavar when nargs=? def test_nargs_optional_metavar_string(self): self.do_test_no_exception(nargs="?", metavar="1") def test_nargs_optional_metavar_length0(self): self.do_test_exception(nargs="?", metavar=tuple()) def test_nargs_optional_metavar_length1(self): self.do_test_no_exception(nargs="?", metavar=("1")) def test_nargs_optional_metavar_length2(self): self.do_test_exception(nargs="?", metavar=("1", "2")) def test_nargs_optional_metavar_length3(self): self.do_test_exception(nargs="?", metavar=("1", "2", "3")) # Unit tests for different values of metavar when nargs=* def test_nargs_zeroormore_metavar_string(self): self.do_test_no_exception(nargs="*", metavar="1") def test_nargs_zeroormore_metavar_length0(self): self.do_test_exception(nargs="*", metavar=tuple()) def test_nargs_zeroormore_metavar_length1(self): self.do_test_no_exception(nargs="*", metavar=("1")) def test_nargs_zeroormore_metavar_length2(self): self.do_test_no_exception(nargs="*", metavar=("1", "2")) def test_nargs_zeroormore_metavar_length3(self): self.do_test_exception(nargs="*", metavar=("1", "2", "3")) # Unit tests for different values of metavar when nargs=+ def test_nargs_oneormore_metavar_string(self): self.do_test_no_exception(nargs="+", metavar="1") def test_nargs_oneormore_metavar_length0(self): self.do_test_exception(nargs="+", metavar=tuple()) def test_nargs_oneormore_metavar_length1(self): self.do_test_no_exception(nargs="+", metavar=("1")) def test_nargs_oneormore_metavar_length2(self): self.do_test_no_exception(nargs="+", metavar=("1", "2")) def test_nargs_oneormore_metavar_length3(self): self.do_test_exception(nargs="+", metavar=("1", "2", "3")) # Unit tests for different values of metavar when nargs=... def test_nargs_remainder_metavar_string(self): self.do_test_no_exception(nargs="...", metavar="1") def test_nargs_remainder_metavar_length0(self): self.do_test_no_exception(nargs="...", metavar=tuple()) def test_nargs_remainder_metavar_length1(self): self.do_test_no_exception(nargs="...", metavar=("1")) def test_nargs_remainder_metavar_length2(self): self.do_test_no_exception(nargs="...", metavar=("1", "2")) def test_nargs_remainder_metavar_length3(self): self.do_test_no_exception(nargs="...", metavar=("1", "2", "3")) # Unit tests for different values of metavar when nargs=A... def test_nargs_parser_metavar_string(self): self.do_test_no_exception(nargs="A...", metavar="1") def test_nargs_parser_metavar_length0(self): self.do_test_exception(nargs="A...", metavar=tuple()) def test_nargs_parser_metavar_length1(self): self.do_test_no_exception(nargs="A...", metavar=("1")) def test_nargs_parser_metavar_length2(self): self.do_test_exception(nargs="A...", metavar=("1", "2")) def test_nargs_parser_metavar_length3(self): self.do_test_exception(nargs="A...", metavar=("1", "2", "3")) # Unit tests for different values of metavar when nargs=1 def test_nargs_1_metavar_string(self): self.do_test_no_exception(nargs=1, metavar="1") def test_nargs_1_metavar_length0(self): self.do_test_exception(nargs=1, metavar=tuple()) def test_nargs_1_metavar_length1(self): self.do_test_no_exception(nargs=1, metavar=("1")) def test_nargs_1_metavar_length2(self): self.do_test_exception(nargs=1, metavar=("1", "2")) def test_nargs_1_metavar_length3(self): self.do_test_exception(nargs=1, metavar=("1", "2", "3")) # Unit tests for different values of metavar when nargs=2 def test_nargs_2_metavar_string(self): self.do_test_no_exception(nargs=2, metavar="1") def test_nargs_2_metavar_length0(self): self.do_test_exception(nargs=2, metavar=tuple()) def test_nargs_2_metavar_length1(self): self.do_test_no_exception(nargs=2, metavar=("1")) def test_nargs_2_metavar_length2(self): self.do_test_no_exception(nargs=2, metavar=("1", "2")) def test_nargs_2_metavar_length3(self): self.do_test_exception(nargs=2, metavar=("1", "2", "3")) # Unit tests for different values of metavar when nargs=3 def test_nargs_3_metavar_string(self): self.do_test_no_exception(nargs=3, metavar="1") def test_nargs_3_metavar_length0(self): self.do_test_exception(nargs=3, metavar=tuple()) def test_nargs_3_metavar_length1(self): self.do_test_no_exception(nargs=3, metavar=("1")) def test_nargs_3_metavar_length2(self): self.do_test_exception(nargs=3, metavar=("1", "2")) def test_nargs_3_metavar_length3(self): self.do_test_no_exception(nargs=3, metavar=("1", "2", "3")) # ============================ # from argparse import * tests # ============================ # class TestImportStar(TestCase): # def test(self): # for name in argparse.__all__: # self.assertTrue(hasattr(argparse, name)) # def test_all_exports_everything_but_modules(self): # items = [ # name # for name, value in vars(argparse).items() # if not name.startswith("_") # if not inspect.ismodule(value) # ] # self.assertEqual(sorted(items), sorted(argparse.__all__)) def test_main(): # silence warnings about version argument - these are expected # with test_support.check_warnings( # ('The "version" argument to ArgumentParser is deprecated.', # DeprecationWarning), # ('The (format|print)_version method is deprecated', # DeprecationWarning)): # test_support.run_unittest(__name__) test_support.run_unittest(__name__) # Remove global references to avoid looking like we have refleaks. RFile.seen = {} WFile.seen = set() if __name__ == '__main__': test_main() ================================================ FILE: third_party/stdlib/test/test_bisect.py ================================================ import sys import unittest from test import test_support #from UserList import UserList import UserList as _UserList UserList = _UserList.UserList # We do a bit of trickery here to be able to test both the C implementation # and the Python implementation of the module. # Make it impossible to import the C implementation anymore. sys.modules['_bisect'] = 0 # We must also handle the case that bisect was imported before. if 'bisect' in sys.modules: del sys.modules['bisect'] # Now we can import the module and get the pure Python implementation. import bisect as py_bisect # Restore everything to normal. del sys.modules['_bisect'] del sys.modules['bisect'] # This is now the module with the C implementation. #import bisect as c_bisect class Range(object): """A trivial xrange()-like object without any integer width limitations.""" def __init__(self, start, stop): self.start = start self.stop = stop self.last_insert = None def __len__(self): return self.stop - self.start def __getitem__(self, idx): n = self.stop - self.start if idx < 0: idx += n if idx >= n: raise IndexError(idx) return self.start + idx def insert(self, idx, item): self.last_insert = idx, item class TestBisect(unittest.TestCase): # module = None module = py_bisect def setUp(self): self.precomputedCases = [ (self.module.bisect_right, [], 1, 0), (self.module.bisect_right, [1], 0, 0), (self.module.bisect_right, [1], 1, 1), (self.module.bisect_right, [1], 2, 1), (self.module.bisect_right, [1, 1], 0, 0), (self.module.bisect_right, [1, 1], 1, 2), (self.module.bisect_right, [1, 1], 2, 2), (self.module.bisect_right, [1, 1, 1], 0, 0), (self.module.bisect_right, [1, 1, 1], 1, 3), (self.module.bisect_right, [1, 1, 1], 2, 3), (self.module.bisect_right, [1, 1, 1, 1], 0, 0), (self.module.bisect_right, [1, 1, 1, 1], 1, 4), (self.module.bisect_right, [1, 1, 1, 1], 2, 4), (self.module.bisect_right, [1, 2], 0, 0), (self.module.bisect_right, [1, 2], 1, 1), (self.module.bisect_right, [1, 2], 1.5, 1), (self.module.bisect_right, [1, 2], 2, 2), (self.module.bisect_right, [1, 2], 3, 2), (self.module.bisect_right, [1, 1, 2, 2], 0, 0), (self.module.bisect_right, [1, 1, 2, 2], 1, 2), (self.module.bisect_right, [1, 1, 2, 2], 1.5, 2), (self.module.bisect_right, [1, 1, 2, 2], 2, 4), (self.module.bisect_right, [1, 1, 2, 2], 3, 4), (self.module.bisect_right, [1, 2, 3], 0, 0), (self.module.bisect_right, [1, 2, 3], 1, 1), (self.module.bisect_right, [1, 2, 3], 1.5, 1), (self.module.bisect_right, [1, 2, 3], 2, 2), (self.module.bisect_right, [1, 2, 3], 2.5, 2), (self.module.bisect_right, [1, 2, 3], 3, 3), (self.module.bisect_right, [1, 2, 3], 4, 3), (self.module.bisect_right, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 0, 0), (self.module.bisect_right, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1, 1), (self.module.bisect_right, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1.5, 1), (self.module.bisect_right, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2, 3), (self.module.bisect_right, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2.5, 3), (self.module.bisect_right, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3, 6), (self.module.bisect_right, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3.5, 6), (self.module.bisect_right, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 4, 10), (self.module.bisect_right, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 5, 10), (self.module.bisect_left, [], 1, 0), (self.module.bisect_left, [1], 0, 0), (self.module.bisect_left, [1], 1, 0), (self.module.bisect_left, [1], 2, 1), (self.module.bisect_left, [1, 1], 0, 0), (self.module.bisect_left, [1, 1], 1, 0), (self.module.bisect_left, [1, 1], 2, 2), (self.module.bisect_left, [1, 1, 1], 0, 0), (self.module.bisect_left, [1, 1, 1], 1, 0), (self.module.bisect_left, [1, 1, 1], 2, 3), (self.module.bisect_left, [1, 1, 1, 1], 0, 0), (self.module.bisect_left, [1, 1, 1, 1], 1, 0), (self.module.bisect_left, [1, 1, 1, 1], 2, 4), (self.module.bisect_left, [1, 2], 0, 0), (self.module.bisect_left, [1, 2], 1, 0), (self.module.bisect_left, [1, 2], 1.5, 1), (self.module.bisect_left, [1, 2], 2, 1), (self.module.bisect_left, [1, 2], 3, 2), (self.module.bisect_left, [1, 1, 2, 2], 0, 0), (self.module.bisect_left, [1, 1, 2, 2], 1, 0), (self.module.bisect_left, [1, 1, 2, 2], 1.5, 2), (self.module.bisect_left, [1, 1, 2, 2], 2, 2), (self.module.bisect_left, [1, 1, 2, 2], 3, 4), (self.module.bisect_left, [1, 2, 3], 0, 0), (self.module.bisect_left, [1, 2, 3], 1, 0), (self.module.bisect_left, [1, 2, 3], 1.5, 1), (self.module.bisect_left, [1, 2, 3], 2, 1), (self.module.bisect_left, [1, 2, 3], 2.5, 2), (self.module.bisect_left, [1, 2, 3], 3, 2), (self.module.bisect_left, [1, 2, 3], 4, 3), (self.module.bisect_left, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 0, 0), (self.module.bisect_left, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1, 0), (self.module.bisect_left, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 1.5, 1), (self.module.bisect_left, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2, 1), (self.module.bisect_left, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 2.5, 3), (self.module.bisect_left, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3, 3), (self.module.bisect_left, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 3.5, 6), (self.module.bisect_left, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 4, 6), (self.module.bisect_left, [1, 2, 2, 3, 3, 3, 4, 4, 4, 4], 5, 10) ] def test_precomputed(self): for func, data, elem, expected in self.precomputedCases: self.assertEqual(func(data, elem), expected) self.assertEqual(func(UserList(data), elem), expected) def test_negative_lo(self): # Issue 3301 mod = self.module self.assertRaises(ValueError, mod.bisect_left, [1, 2, 3], 5, -1, 3), self.assertRaises(ValueError, mod.bisect_right, [1, 2, 3], 5, -1, 3), self.assertRaises(ValueError, mod.insort_left, [1, 2, 3], 5, -1, 3), self.assertRaises(ValueError, mod.insort_right, [1, 2, 3], 5, -1, 3), def test_large_range(self): # Issue 13496 mod = self.module n = sys.maxsize try: data = xrange(n-1) except OverflowError: self.skipTest("can't create a xrange() object of size `sys.maxsize`") self.assertEqual(mod.bisect_left(data, n-3), n-3) self.assertEqual(mod.bisect_right(data, n-3), n-2) self.assertEqual(mod.bisect_left(data, n-3, n-10, n), n-3) self.assertEqual(mod.bisect_right(data, n-3, n-10, n), n-2) def test_large_pyrange(self): # Same as above, but without C-imposed limits on range() parameters mod = self.module n = sys.maxsize data = Range(0, n-1) self.assertEqual(mod.bisect_left(data, n-3), n-3) self.assertEqual(mod.bisect_right(data, n-3), n-2) self.assertEqual(mod.bisect_left(data, n-3, n-10, n), n-3) self.assertEqual(mod.bisect_right(data, n-3, n-10, n), n-2) x = n - 100 mod.insort_left(data, x, x - 50, x + 50) self.assertEqual(data.last_insert, (x, x)) x = n - 200 mod.insort_right(data, x, x - 50, x + 50) self.assertEqual(data.last_insert, (x + 1, x)) def test_random(self, n=25): #from random import randrange import random as _random randrange = _random.randrange for i in xrange(n): data = [randrange(0, n, 2) for j in xrange(i)] data.sort() elem = randrange(-1, n+1) ip = self.module.bisect_left(data, elem) if ip < len(data): self.assertTrue(elem <= data[ip]) if ip > 0: self.assertTrue(data[ip-1] < elem) ip = self.module.bisect_right(data, elem) if ip < len(data): self.assertTrue(elem < data[ip]) if ip > 0: self.assertTrue(data[ip-1] <= elem) def test_optionalSlicing(self): for func, data, elem, expected in self.precomputedCases: for lo in xrange(4): lo = min(len(data), lo) for hi in xrange(3,8): hi = min(len(data), hi) ip = func(data, elem, lo, hi) self.assertTrue(lo <= ip <= hi) if func is self.module.bisect_left and ip < hi: self.assertTrue(elem <= data[ip]) if func is self.module.bisect_left and ip > lo: self.assertTrue(data[ip-1] < elem) if func is self.module.bisect_right and ip < hi: self.assertTrue(elem < data[ip]) if func is self.module.bisect_right and ip > lo: self.assertTrue(data[ip-1] <= elem) self.assertEqual(ip, max(lo, min(hi, expected))) def test_backcompatibility(self): self.assertEqual(self.module.bisect, self.module.bisect_right) def test_keyword_args(self): data = [10, 20, 30, 40, 50] self.assertEqual(self.module.bisect_left(a=data, x=25, lo=1, hi=3), 2) self.assertEqual(self.module.bisect_right(a=data, x=25, lo=1, hi=3), 2) self.assertEqual(self.module.bisect(a=data, x=25, lo=1, hi=3), 2) self.module.insort_left(a=data, x=25, lo=1, hi=3) self.module.insort_right(a=data, x=25, lo=1, hi=3) self.module.insort(a=data, x=25, lo=1, hi=3) self.assertEqual(data, [10, 20, 25, 25, 25, 30, 40, 50]) # class TestBisectPython(TestBisect): # module = py_bisect # class TestBisectC(TestBisect): # module = c_bisect #============================================================================== class TestInsort(unittest.TestCase): # module = None module = py_bisect def test_vsBuiltinSort(self, n=500): #from random import choice import random as _random choice = _random.choice for insorted in (list(), UserList()): for i in xrange(n): digit = choice("0123456789") if digit in "02468": f = self.module.insort_left else: f = self.module.insort_right f(insorted, digit) self.assertEqual(sorted(insorted), insorted) def test_backcompatibility(self): self.assertEqual(self.module.insort, self.module.insort_right) def test_listDerived(self): class List(list): data = [] def insert(self, index, item): self.data.insert(index, item) lst = List() self.module.insort_left(lst, 10) self.module.insort_right(lst, 5) self.assertEqual([5, 10], lst.data) # class TestInsortPython(TestInsort): # module = py_bisect # class TestInsortC(TestInsort): # module = c_bisect #============================================================================== class LenOnly(object): "Dummy sequence class defining __len__ but not __getitem__." def __len__(self): return 10 # Have to define LenOnly as an object for the Grumpy runtime. Doing so will # raise a TypeError instead of an AttributeError when accessing __getitem__, # so we redefine __getitem__ to raise an AttributeError. def __getitem__(self, ndx): raise AttributeError class GetOnly(object): "Dummy sequence class defining __getitem__ but not __len__." def __getitem__(self, ndx): return 10 def __len__(self): raise AttributeError class CmpErr(object): "Dummy element that always raises an error during comparison" def __cmp__(self, other): raise ZeroDivisionError class TestErrorHandling(unittest.TestCase): # module = None module = py_bisect def test_non_sequence(self): for f in (self.module.bisect_left, self.module.bisect_right, self.module.insort_left, self.module.insort_right): self.assertRaises(TypeError, f, 10, 10) def test_len_only(self): for f in (self.module.bisect_left, self.module.bisect_right, self.module.insort_left, self.module.insort_right): self.assertRaises(AttributeError, f, LenOnly(), 10) def test_get_only(self): for f in (self.module.bisect_left, self.module.bisect_right, self.module.insort_left, self.module.insort_right): self.assertRaises(AttributeError, f, GetOnly(), 10) def test_cmp_err(self): seq = [CmpErr(), CmpErr(), CmpErr()] for f in (self.module.bisect_left, self.module.bisect_right, self.module.insort_left, self.module.insort_right): self.assertRaises(ZeroDivisionError, f, seq, 10) def test_arg_parsing(self): for f in (self.module.bisect_left, self.module.bisect_right, self.module.insort_left, self.module.insort_right): self.assertRaises(TypeError, f, 10) # class TestErrorHandlingPython(TestErrorHandling): # module = py_bisect # class TestErrorHandlingC(TestErrorHandling): # module = c_bisect #============================================================================== libreftest = """ Example from the Library Reference: Doc/library/bisect.rst The bisect() function is generally useful for categorizing numeric data. This example uses bisect() to look up a letter grade for an exam total (say) based on a set of ordered numeric breakpoints: 85 and up is an `A', 75..84 is a `B', etc. >>> grades = "FEDCBA" >>> breakpoints = [30, 44, 66, 75, 85] >>> from bisect import bisect >>> def grade(total): ... return grades[bisect(breakpoints, total)] ... >>> grade(66) 'C' >>> map(grade, [33, 99, 77, 44, 12, 88]) ['E', 'A', 'B', 'D', 'F', 'A'] """ #------------------------------------------------------------------------------ __test__ = {'libreftest' : libreftest} def test_main(verbose=None): # from test import test_bisect # test_classes = [TestBisectPython, TestBisectC, # TestInsortPython, TestInsortC, # TestErrorHandlingPython, TestErrorHandlingC] test_classes = [TestBisect, TestInsort, TestErrorHandling] test_support.run_unittest(*test_classes) # test_support.run_doctest(test_bisect, verbose) # verify reference counting if verbose and hasattr(sys, "gettotalrefcount"): #import gc counts = [None] * 5 for i in xrange(len(counts)): test_support.run_unittest(*test_classes) #gc.collect() counts[i] = sys.gettotalrefcount() print counts if __name__ == "__main__": test_main(verbose=True) ================================================ FILE: third_party/stdlib/test/test_colorsys.py ================================================ import unittest import colorsys from test import test_support def frange(start, stop, step): while start <= stop: yield start start += step class ColorsysTest(unittest.TestCase): def assertTripleEqual(self, tr1, tr2): self.assertEqual(len(tr1), 3) self.assertEqual(len(tr2), 3) self.assertAlmostEqual(tr1[0], tr2[0]) self.assertAlmostEqual(tr1[1], tr2[1]) self.assertAlmostEqual(tr1[2], tr2[2]) def test_hsv_roundtrip(self): for r in frange(0.0, 1.0, 0.2): for g in frange(0.0, 1.0, 0.2): for b in frange(0.0, 1.0, 0.2): rgb = (r, g, b) self.assertTripleEqual( rgb, colorsys.hsv_to_rgb(*colorsys.rgb_to_hsv(*rgb)) ) def test_hsv_values(self): values = [ # rgb, hsv ((0.0, 0.0, 0.0), ( 0 , 0.0, 0.0)), # black ((0.0, 0.0, 1.0), (4./6., 1.0, 1.0)), # blue ((0.0, 1.0, 0.0), (2./6., 1.0, 1.0)), # green ((0.0, 1.0, 1.0), (3./6., 1.0, 1.0)), # cyan ((1.0, 0.0, 0.0), ( 0 , 1.0, 1.0)), # red ((1.0, 0.0, 1.0), (5./6., 1.0, 1.0)), # purple ((1.0, 1.0, 0.0), (1./6., 1.0, 1.0)), # yellow ((1.0, 1.0, 1.0), ( 0 , 0.0, 1.0)), # white ((0.5, 0.5, 0.5), ( 0 , 0.0, 0.5)), # grey ] for (rgb, hsv) in values: self.assertTripleEqual(hsv, colorsys.rgb_to_hsv(*rgb)) self.assertTripleEqual(rgb, colorsys.hsv_to_rgb(*hsv)) def test_hls_roundtrip(self): for r in frange(0.0, 1.0, 0.2): for g in frange(0.0, 1.0, 0.2): for b in frange(0.0, 1.0, 0.2): rgb = (r, g, b) self.assertTripleEqual( rgb, colorsys.hls_to_rgb(*colorsys.rgb_to_hls(*rgb)) ) def test_hls_values(self): values = [ # rgb, hls ((0.0, 0.0, 0.0), ( 0 , 0.0, 0.0)), # black ((0.0, 0.0, 1.0), (4./6., 0.5, 1.0)), # blue ((0.0, 1.0, 0.0), (2./6., 0.5, 1.0)), # green ((0.0, 1.0, 1.0), (3./6., 0.5, 1.0)), # cyan ((1.0, 0.0, 0.0), ( 0 , 0.5, 1.0)), # red ((1.0, 0.0, 1.0), (5./6., 0.5, 1.0)), # purple ((1.0, 1.0, 0.0), (1./6., 0.5, 1.0)), # yellow ((1.0, 1.0, 1.0), ( 0 , 1.0, 0.0)), # white ((0.5, 0.5, 0.5), ( 0 , 0.5, 0.0)), # grey ] for (rgb, hls) in values: self.assertTripleEqual(hls, colorsys.rgb_to_hls(*rgb)) self.assertTripleEqual(rgb, colorsys.hls_to_rgb(*hls)) def test_yiq_roundtrip(self): for r in frange(0.0, 1.0, 0.2): for g in frange(0.0, 1.0, 0.2): for b in frange(0.0, 1.0, 0.2): rgb = (r, g, b) self.assertTripleEqual( rgb, colorsys.yiq_to_rgb(*colorsys.rgb_to_yiq(*rgb)) ) def test_yiq_values(self): values = [ # rgb, yiq ((0.0, 0.0, 0.0), (0.0, 0.0, 0.0)), # black ((0.0, 0.0, 1.0), (0.11, -0.3217, 0.3121)), # blue ((0.0, 1.0, 0.0), (0.59, -0.2773, -0.5251)), # green ((0.0, 1.0, 1.0), (0.7, -0.599, -0.213)), # cyan ((1.0, 0.0, 0.0), (0.3, 0.599, 0.213)), # red ((1.0, 0.0, 1.0), (0.41, 0.2773, 0.5251)), # purple ((1.0, 1.0, 0.0), (0.89, 0.3217, -0.3121)), # yellow ((1.0, 1.0, 1.0), (1.0, 0.0, 0.0)), # white ((0.5, 0.5, 0.5), (0.5, 0.0, 0.0)), # grey ] for (rgb, yiq) in values: self.assertTripleEqual(yiq, colorsys.rgb_to_yiq(*rgb)) self.assertTripleEqual(rgb, colorsys.yiq_to_rgb(*yiq)) def test_main(): test_support.run_unittest(ColorsysTest) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_datetime.py ================================================ """Test date/time type. See http://www.zope.org/Members/fdrake/DateTimeWiki/TestCases """ # from __future__ import division import sys # import pickle # import cPickle import unittest from test import test_support # from datetime import MINYEAR, MAXYEAR # from datetime import timedelta # from datetime import tzinfo # from datetime import time # from datetime import date, datetime import datetime MINYEAR, MAXYEAR, timedelta, tzinfo, time, date, datetime = \ datetime.MINYEAR, datetime.MAXYEAR, datetime.timedelta, datetime.tzinfo, \ datetime.time, datetime.date, datetime.datetime # pickle_choices = [(pickler, unpickler, proto) # for pickler in pickle, cPickle # for unpickler in pickle, cPickle # for proto in range(3)] # assert len(pickle_choices) == 2*2*3 # An arbitrary collection of objects of non-datetime types, for testing # mixed-type comparisons. OTHERSTUFF = (10, 10L, 34.5, "abc", {}, [], ()) ############################################################################# # module tests class TestModule(unittest.TestCase): def test_constants(self): import datetime self.assertEqual(datetime.MINYEAR, 1) self.assertEqual(datetime.MAXYEAR, 9999) ############################################################################# # tzinfo tests class FixedOffset(tzinfo): def __init__(self, offset, name, dstoffset=42): if isinstance(offset, int): offset = timedelta(minutes=offset) if isinstance(dstoffset, int): dstoffset = timedelta(minutes=dstoffset) self.__offset = offset self.__name = name self.__dstoffset = dstoffset def __repr__(self): return self.__name.lower() def utcoffset(self, dt): return self.__offset def tzname(self, dt): return self.__name def dst(self, dt): return self.__dstoffset class PicklableFixedOffset(FixedOffset): def __init__(self, offset=None, name=None, dstoffset=None): FixedOffset.__init__(self, offset, name, dstoffset) class TestTZInfo(unittest.TestCase): def test_non_abstractness(self): # In order to allow subclasses to get pickled, the C implementation # wasn't able to get away with having __init__ raise # NotImplementedError. useless = tzinfo() dt = datetime.max self.assertRaises(NotImplementedError, useless.tzname, dt) self.assertRaises(NotImplementedError, useless.utcoffset, dt) self.assertRaises(NotImplementedError, useless.dst, dt) def test_subclass_must_override(self): class NotEnough(tzinfo): def __init__(self, offset, name): self.__offset = offset self.__name = name self.assertTrue(issubclass(NotEnough, tzinfo)) ne = NotEnough(3, "NotByALongShot") self.assertIsInstance(ne, tzinfo) dt = datetime.now() self.assertRaises(NotImplementedError, ne.tzname, dt) self.assertRaises(NotImplementedError, ne.utcoffset, dt) self.assertRaises(NotImplementedError, ne.dst, dt) def test_normal(self): fo = FixedOffset(3, "Three") self.assertIsInstance(fo, tzinfo) for dt in datetime.now(), None: self.assertEqual(fo.utcoffset(dt), timedelta(minutes=3)) self.assertEqual(fo.tzname(dt), "Three") self.assertEqual(fo.dst(dt), timedelta(minutes=42)) # def test_pickling_base(self): # # There's no point to pickling tzinfo objects on their own (they # # carry no data), but they need to be picklable anyway else # # concrete subclasses can't be pickled. # orig = tzinfo.__new__(tzinfo) # self.assertIs(type(orig), tzinfo) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertIs(type(derived), tzinfo) # def test_pickling_subclass(self): # # Make sure we can pickle/unpickle an instance of a subclass. # offset = timedelta(minutes=-300) # orig = PicklableFixedOffset(offset, 'cookie') # self.assertIsInstance(orig, tzinfo) # self.assertTrue(type(orig) is PicklableFixedOffset) # self.assertEqual(orig.utcoffset(None), offset) # self.assertEqual(orig.tzname(None), 'cookie') # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertIsInstance(derived, tzinfo) # self.assertTrue(type(derived) is PicklableFixedOffset) # self.assertEqual(derived.utcoffset(None), offset) # self.assertEqual(derived.tzname(None), 'cookie') ############################################################################# # Base class for testing a particular aspect of timedelta, time, date and # datetime comparisons. class HarmlessMixedComparison(object): # Test that __eq__ and __ne__ don't complain for mixed-type comparisons. # Subclasses must define 'theclass', and theclass(1, 1, 1) must be a # legit constructor. def test_harmless_mixed_comparison(self): me = self.theclass(1, 1, 1) self.assertFalse(me == ()) self.assertTrue(me != ()) self.assertFalse(() == me) self.assertTrue(() != me) self.assertIn(me, [1, 20L, [], me]) self.assertIn([], [me, 1, 20L, []]) def test_harmful_mixed_comparison(self): me = self.theclass(1, 1, 1) self.assertRaises(TypeError, lambda: me < ()) self.assertRaises(TypeError, lambda: me <= ()) self.assertRaises(TypeError, lambda: me > ()) self.assertRaises(TypeError, lambda: me >= ()) self.assertRaises(TypeError, lambda: () < me) self.assertRaises(TypeError, lambda: () <= me) self.assertRaises(TypeError, lambda: () > me) self.assertRaises(TypeError, lambda: () >= me) self.assertRaises(TypeError, cmp, (), me) self.assertRaises(TypeError, cmp, me, ()) ############################################################################# # timedelta tests class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): theclass = timedelta def test_constructor(self): eq = self.assertEqual td = timedelta # Check keyword args to constructor eq(td(), td(weeks=0, days=0, hours=0, minutes=0, seconds=0, milliseconds=0, microseconds=0)) eq(td(1), td(days=1)) eq(td(0, 1), td(seconds=1)) eq(td(0, 0, 1), td(microseconds=1)) eq(td(weeks=1), td(days=7)) eq(td(days=1), td(hours=24)) eq(td(hours=1), td(minutes=60)) eq(td(minutes=1), td(seconds=60)) eq(td(seconds=1), td(milliseconds=1000)) eq(td(milliseconds=1), td(microseconds=1000)) # Check float args to constructor eq(td(weeks=1.0/7), td(days=1)) eq(td(days=1.0/24), td(hours=1)) eq(td(hours=1.0/60), td(minutes=1)) eq(td(minutes=1.0/60), td(seconds=1)) eq(td(seconds=0.001), td(milliseconds=1)) eq(td(milliseconds=0.001), td(microseconds=1)) def test_computations(self): eq = self.assertEqual td = timedelta a = td(7) # One week b = td(0, 60) # One minute c = td(0, 0, 1000) # One millisecond eq(a+b+c, td(7, 60, 1000)) eq(a-b, td(6, 24*3600 - 60)) eq(-a, td(-7)) # eq(+a, td(7)) eq(-b, td(-1, 24*3600 - 60)) eq(-c, td(-1, 24*3600 - 1, 999000)) eq(abs(a), a) eq(abs(-a), a) eq(td(6, 24*3600), a) eq(td(0, 0, 60*1000000), b) eq(a*10, td(70)) eq(a*10, 10*a) eq(a*10L, 10*a) eq(b*10, td(0, 600)) eq(10*b, td(0, 600)) eq(b*10L, td(0, 600)) eq(c*10, td(0, 0, 10000)) eq(10*c, td(0, 0, 10000)) eq(c*10L, td(0, 0, 10000)) eq(a*-1, -a) eq(b*-2, -b-b) eq(c*-2, -c+-c) eq(b*(60*24), (b*60)*24) eq(b*(60*24), (60*b)*24) eq(c*1000, td(0, 1)) eq(1000*c, td(0, 1)) eq(a//7, td(1)) eq(b//10, td(0, 6)) eq(c//1000, td(0, 0, 1)) eq(a//10, td(0, 7*24*360)) eq(a//3600000, td(0, 0, 7*24*1000)) # Issue #11576 eq(td(999999999, 86399, 999999) - td(999999999, 86399, 999998), td(0, 0, 1)) eq(td(999999999, 1, 1) - td(999999999, 1, 0), td(0, 0, 1)) def test_disallowed_computations(self): a = timedelta(42) # Add/sub ints, longs, floats should be illegal for i in 1, 1L, 1.0: self.assertRaises(TypeError, lambda: a+i) self.assertRaises(TypeError, lambda: a-i) self.assertRaises(TypeError, lambda: i+a) self.assertRaises(TypeError, lambda: i-a) # Mul/div by float isn't supported. x = 2.3 self.assertRaises(TypeError, lambda: a*x) self.assertRaises(TypeError, lambda: x*a) self.assertRaises(TypeError, lambda: a/x) self.assertRaises(TypeError, lambda: x/a) self.assertRaises(TypeError, lambda: a // x) self.assertRaises(TypeError, lambda: x // a) # Division of int by timedelta doesn't make sense. # Division by zero doesn't make sense. for zero in 0, 0L: self.assertRaises(TypeError, lambda: zero // a) self.assertRaises(ZeroDivisionError, lambda: a // zero) def test_basic_attributes(self): days, seconds, us = 1, 7, 31 td = timedelta(days, seconds, us) self.assertEqual(td.days, days) self.assertEqual(td.seconds, seconds) self.assertEqual(td.microseconds, us) @unittest.expectedFailure def test_total_seconds(self): td = timedelta(days=365) self.assertEqual(td.total_seconds(), 31536000.0) for total_seconds in [123456.789012, -123456.789012, 0.123456, 0, 1e6]: td = timedelta(seconds=total_seconds) self.assertEqual(td.total_seconds(), total_seconds) # Issue8644: Test that td.total_seconds() has the same # accuracy as td / timedelta(seconds=1). for ms in [-1, -2, -123]: td = timedelta(microseconds=ms) self.assertEqual(td.total_seconds(), ((24*3600*td.days + td.seconds)*10**6 + td.microseconds)/10**6) def test_carries(self): t1 = timedelta(days=100, weeks=-7, hours=-24*(100-49), minutes=-3, seconds=12, microseconds=(3*60 - 12) * 1e6 + 1) t2 = timedelta(microseconds=1) self.assertEqual(t1, t2) @unittest.expectedFailure def test_hash_equality(self): t1 = timedelta(days=100, weeks=-7, hours=-24*(100-49), minutes=-3, seconds=12, microseconds=(3*60 - 12) * 1000000) t2 = timedelta() self.assertEqual(hash(t1), hash(t2)) t1 += timedelta(weeks=7) t2 += timedelta(days=7*7) self.assertEqual(t1, t2) self.assertEqual(hash(t1), hash(t2)) d = {t1: 1} d[t2] = 2 self.assertEqual(len(d), 1) self.assertEqual(d[t1], 2) # def test_pickling(self): # args = 12, 34, 56 # orig = timedelta(*args) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) def test_compare(self): t1 = timedelta(2, 3, 4) t2 = timedelta(2, 3, 4) self.assertTrue(t1 == t2) self.assertTrue(t1 <= t2) self.assertTrue(t1 >= t2) self.assertFalse(t1 != t2) self.assertFalse(t1 < t2) self.assertFalse(t1 > t2) self.assertEqual(cmp(t1, t2), 0) self.assertEqual(cmp(t2, t1), 0) for args in (3, 3, 3), (2, 4, 4), (2, 3, 5): t2 = timedelta(*args) # this is larger than t1 self.assertTrue(t1 < t2) self.assertTrue(t2 > t1) self.assertTrue(t1 <= t2) self.assertTrue(t2 >= t1) self.assertTrue(t1 != t2) self.assertTrue(t2 != t1) self.assertFalse(t1 == t2) self.assertFalse(t2 == t1) self.assertFalse(t1 > t2) self.assertFalse(t2 < t1) self.assertFalse(t1 >= t2) self.assertFalse(t2 <= t1) self.assertEqual(cmp(t1, t2), -1) self.assertEqual(cmp(t2, t1), 1) for badarg in OTHERSTUFF: self.assertEqual(t1 == badarg, False) self.assertEqual(t1 != badarg, True) self.assertEqual(badarg == t1, False) self.assertEqual(badarg != t1, True) self.assertRaises(TypeError, lambda: t1 <= badarg) self.assertRaises(TypeError, lambda: t1 < badarg) self.assertRaises(TypeError, lambda: t1 > badarg) self.assertRaises(TypeError, lambda: t1 >= badarg) self.assertRaises(TypeError, lambda: badarg <= t1) self.assertRaises(TypeError, lambda: badarg < t1) self.assertRaises(TypeError, lambda: badarg > t1) self.assertRaises(TypeError, lambda: badarg >= t1) def test_str(self): td = timedelta eq = self.assertEqual eq(str(td(1)), "1 day, 0:00:00") eq(str(td(-1)), "-1 day, 0:00:00") eq(str(td(2)), "2 days, 0:00:00") eq(str(td(-2)), "-2 days, 0:00:00") eq(str(td(hours=12, minutes=58, seconds=59)), "12:58:59") eq(str(td(hours=2, minutes=3, seconds=4)), "2:03:04") eq(str(td(weeks=-30, hours=23, minutes=12, seconds=34)), "-210 days, 23:12:34") eq(str(td(milliseconds=1)), "0:00:00.001000") eq(str(td(microseconds=3)), "0:00:00.000003") eq(str(td(days=999999999, hours=23, minutes=59, seconds=59, microseconds=999999)), "999999999 days, 23:59:59.999999") @unittest.expectedFailure def test_roundtrip(self): for td in (timedelta(days=999999999, hours=23, minutes=59, seconds=59, microseconds=999999), timedelta(days=-999999999), timedelta(days=1, seconds=2, microseconds=3)): # Verify td -> string -> td identity. s = repr(td) self.assertTrue(s.startswith('datetime.')) s = s[9:] td2 = eval(s) self.assertEqual(td, td2) # Verify identity via reconstructing from pieces. td2 = timedelta(td.days, td.seconds, td.microseconds) self.assertEqual(td, td2) def test_resolution_info(self): self.assertIsInstance(timedelta.min, timedelta) self.assertIsInstance(timedelta.max, timedelta) self.assertIsInstance(timedelta.resolution, timedelta) self.assertTrue(timedelta.max > timedelta.min) self.assertEqual(timedelta.min, timedelta(-999999999)) self.assertEqual(timedelta.max, timedelta(999999999, 24*3600-1, 1e6-1)) self.assertEqual(timedelta.resolution, timedelta(0, 0, 1)) def test_overflow(self): tiny = timedelta.resolution td = timedelta.min + tiny td -= tiny # no problem self.assertRaises(OverflowError, td.__sub__, tiny) self.assertRaises(OverflowError, td.__add__, -tiny) td = timedelta.max - tiny td += tiny # no problem self.assertRaises(OverflowError, td.__add__, tiny) self.assertRaises(OverflowError, td.__sub__, -tiny) self.assertRaises(OverflowError, lambda: -timedelta.max) def test_microsecond_rounding(self): td = timedelta eq = self.assertEqual # Single-field rounding. eq(td(milliseconds=0.4/1000), td(0)) # rounds to 0 eq(td(milliseconds=-0.4/1000), td(0)) # rounds to 0 eq(td(milliseconds=0.6/1000), td(microseconds=1)) eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 us_per_day = us_per_hour * 24 eq(td(days=.4/us_per_day), td(0)) eq(td(hours=.2/us_per_hour), td(0)) eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1)) eq(td(days=-.4/us_per_day), td(0)) eq(td(hours=-.2/us_per_hour), td(0)) eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1)) def test_massive_normalization(self): td = timedelta(microseconds=-1) self.assertEqual((td.days, td.seconds, td.microseconds), (-1, 24*3600-1, 999999)) def test_bool(self): self.assertTrue(timedelta(1)) self.assertTrue(timedelta(0, 1)) self.assertTrue(timedelta(0, 0, 1)) self.assertTrue(timedelta(microseconds=1)) self.assertFalse(timedelta(0)) def test_subclass_timedelta(self): class T(timedelta): @staticmethod def from_td(td): return T(td.days, td.seconds, td.microseconds) def as_hours(self): sum = (self.days * 24 + self.seconds / 3600.0 + self.microseconds / 3600e6) return round(sum) t1 = T(days=1) self.assertIs(type(t1), T) self.assertEqual(t1.as_hours(), 24) t2 = T(days=-1, seconds=-3600) self.assertIs(type(t2), T) self.assertEqual(t2.as_hours(), -25) t3 = t1 + t2 self.assertIs(type(t3), timedelta) t4 = T.from_td(t3) self.assertIs(type(t4), T) self.assertEqual(t3.days, t4.days) self.assertEqual(t3.seconds, t4.seconds) self.assertEqual(t3.microseconds, t4.microseconds) self.assertEqual(str(t3), str(t4)) self.assertEqual(t4.as_hours(), -1) ############################################################################# # date tests class TestDateOnly(unittest.TestCase): # Tests here won't pass if also run on datetime objects, so don't # subclass this to test datetimes too. def test_delta_non_days_ignored(self): dt = date(2000, 1, 2) delta = timedelta(days=1, hours=2, minutes=3, seconds=4, microseconds=5) days = timedelta(delta.days) self.assertEqual(days, timedelta(1)) dt2 = dt + delta self.assertEqual(dt2, dt + days) dt2 = delta + dt self.assertEqual(dt2, dt + days) dt2 = dt - delta self.assertEqual(dt2, dt - days) delta = -delta days = timedelta(delta.days) self.assertEqual(days, timedelta(-2)) dt2 = dt + delta self.assertEqual(dt2, dt + days) dt2 = delta + dt self.assertEqual(dt2, dt + days) dt2 = dt - delta self.assertEqual(dt2, dt - days) class SubclassDate(date): sub_var = 1 class TestDate(HarmlessMixedComparison, unittest.TestCase): # Tests here should pass for both dates and datetimes, except for a # few tests that TestDateTime overrides. theclass = date def test_basic_attributes(self): dt = self.theclass(2002, 3, 1) self.assertEqual(dt.year, 2002) self.assertEqual(dt.month, 3) self.assertEqual(dt.day, 1) @unittest.expectedFailure def test_roundtrip(self): for dt in (self.theclass(1, 2, 3), self.theclass.today()): # Verify dt -> string -> date identity. s = repr(dt) self.assertTrue(s.startswith('datetime.')) s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) # Verify identity via reconstructing from pieces. dt2 = self.theclass(dt.year, dt.month, dt.day) self.assertEqual(dt, dt2) def test_ordinal_conversions(self): # Check some fixed values. for y, m, d, n in [(1, 1, 1, 1), # calendar origin (1, 12, 31, 365), (2, 1, 1, 366), # first example from "Calendrical Calculations" (1945, 11, 12, 710347)]: d = self.theclass(y, m, d) self.assertEqual(n, d.toordinal()) fromord = self.theclass.fromordinal(n) self.assertEqual(d, fromord) if hasattr(fromord, "hour"): # if we're checking something fancier than a date, verify # the extra fields have been zeroed out self.assertEqual(fromord.hour, 0) self.assertEqual(fromord.minute, 0) self.assertEqual(fromord.second, 0) self.assertEqual(fromord.microsecond, 0) # Check first and last days of year spottily across the whole # range of years supported. for year in xrange(MINYEAR, MAXYEAR+1, 7): # Verify (year, 1, 1) -> ordinal -> y, m, d is identity. d = self.theclass(year, 1, 1) n = d.toordinal() d2 = self.theclass.fromordinal(n) self.assertEqual(d, d2) # Verify that moving back a day gets to the end of year-1. if year > 1: d = self.theclass.fromordinal(n-1) d2 = self.theclass(year-1, 12, 31) self.assertEqual(d, d2) self.assertEqual(d2.toordinal(), n-1) # Test every day in a leap-year and a non-leap year. dim = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] for year, isleap in (2000, True), (2002, False): n = self.theclass(year, 1, 1).toordinal() for month, maxday in zip(range(1, 13), dim): if month == 2 and isleap: maxday += 1 for day in range(1, maxday+1): d = self.theclass(year, month, day) self.assertEqual(d.toordinal(), n) self.assertEqual(d, self.theclass.fromordinal(n)) n += 1 def test_extreme_ordinals(self): a = self.theclass.min a = self.theclass(a.year, a.month, a.day) # get rid of time parts aord = a.toordinal() b = a.fromordinal(aord) self.assertEqual(a, b) self.assertRaises(ValueError, lambda: a.fromordinal(aord - 1)) b = a + timedelta(days=1) self.assertEqual(b.toordinal(), aord + 1) self.assertEqual(b, self.theclass.fromordinal(aord + 1)) a = self.theclass.max a = self.theclass(a.year, a.month, a.day) # get rid of time parts aord = a.toordinal() b = a.fromordinal(aord) self.assertEqual(a, b) self.assertRaises(ValueError, lambda: a.fromordinal(aord + 1)) b = a - timedelta(days=1) self.assertEqual(b.toordinal(), aord - 1) self.assertEqual(b, self.theclass.fromordinal(aord - 1)) def test_bad_constructor_arguments(self): # bad years self.theclass(MINYEAR, 1, 1) # no exception self.theclass(MAXYEAR, 1, 1) # no exception self.assertRaises(ValueError, self.theclass, MINYEAR-1, 1, 1) self.assertRaises(ValueError, self.theclass, MAXYEAR+1, 1, 1) # bad months self.theclass(2000, 1, 1) # no exception self.theclass(2000, 12, 1) # no exception self.assertRaises(ValueError, self.theclass, 2000, 0, 1) self.assertRaises(ValueError, self.theclass, 2000, 13, 1) # bad days self.theclass(2000, 2, 29) # no exception self.theclass(2004, 2, 29) # no exception self.theclass(2400, 2, 29) # no exception self.assertRaises(ValueError, self.theclass, 2000, 2, 30) self.assertRaises(ValueError, self.theclass, 2001, 2, 29) self.assertRaises(ValueError, self.theclass, 2100, 2, 29) self.assertRaises(ValueError, self.theclass, 1900, 2, 29) self.assertRaises(ValueError, self.theclass, 2000, 1, 0) self.assertRaises(ValueError, self.theclass, 2000, 1, 32) @unittest.expectedFailure def test_hash_equality(self): d = self.theclass(2000, 12, 31) # same thing e = self.theclass(2000, 12, 31) self.assertEqual(d, e) self.assertEqual(hash(d), hash(e)) dic = {d: 1} dic[e] = 2 self.assertEqual(len(dic), 1) self.assertEqual(dic[d], 2) self.assertEqual(dic[e], 2) d = self.theclass(2001, 1, 1) # same thing e = self.theclass(2001, 1, 1) self.assertEqual(d, e) self.assertEqual(hash(d), hash(e)) dic = {d: 1} dic[e] = 2 self.assertEqual(len(dic), 1) self.assertEqual(dic[d], 2) self.assertEqual(dic[e], 2) def test_computations(self): a = self.theclass(2002, 1, 31) b = self.theclass(1956, 1, 31) diff = a-b self.assertEqual(diff.days, 46*365 + len(range(1956, 2002, 4))) self.assertEqual(diff.seconds, 0) self.assertEqual(diff.microseconds, 0) day = timedelta(1) week = timedelta(7) a = self.theclass(2002, 3, 2) self.assertEqual(a + day, self.theclass(2002, 3, 3)) self.assertEqual(day + a, self.theclass(2002, 3, 3)) self.assertEqual(a - day, self.theclass(2002, 3, 1)) self.assertEqual(-day + a, self.theclass(2002, 3, 1)) self.assertEqual(a + week, self.theclass(2002, 3, 9)) self.assertEqual(a - week, self.theclass(2002, 2, 23)) self.assertEqual(a + 52*week, self.theclass(2003, 3, 1)) self.assertEqual(a - 52*week, self.theclass(2001, 3, 3)) self.assertEqual((a + week) - a, week) self.assertEqual((a + day) - a, day) self.assertEqual((a - week) - a, -week) self.assertEqual((a - day) - a, -day) self.assertEqual(a - (a + week), -week) self.assertEqual(a - (a + day), -day) self.assertEqual(a - (a - week), week) self.assertEqual(a - (a - day), day) # Add/sub ints, longs, floats should be illegal for i in 1, 1L, 1.0: self.assertRaises(TypeError, lambda: a+i) self.assertRaises(TypeError, lambda: a-i) self.assertRaises(TypeError, lambda: i+a) self.assertRaises(TypeError, lambda: i-a) # delta - date is senseless. self.assertRaises(TypeError, lambda: day - a) # mixing date and (delta or date) via * or // is senseless self.assertRaises(TypeError, lambda: day * a) self.assertRaises(TypeError, lambda: a * day) self.assertRaises(TypeError, lambda: day // a) self.assertRaises(TypeError, lambda: a // day) self.assertRaises(TypeError, lambda: a * a) self.assertRaises(TypeError, lambda: a // a) # date + date is senseless self.assertRaises(TypeError, lambda: a + a) def test_overflow(self): tiny = self.theclass.resolution for delta in [tiny, timedelta(1), timedelta(2)]: dt = self.theclass.min + delta dt -= delta # no problem self.assertRaises(OverflowError, dt.__sub__, delta) self.assertRaises(OverflowError, dt.__add__, -delta) dt = self.theclass.max - delta dt += delta # no problem self.assertRaises(OverflowError, dt.__add__, delta) self.assertRaises(OverflowError, dt.__sub__, -delta) def test_fromtimestamp(self): import time # Try an arbitrary fixed value. year, month, day = 1999, 9, 19 ts = time.mktime((year, month, day, 0, 0, 0, 0, 0, -1)) d = self.theclass.fromtimestamp(ts) self.assertEqual(d.year, year) self.assertEqual(d.month, month) self.assertEqual(d.day, day) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, # and that this test will fail there. This test should # exempt such platforms (provided they return reasonable # results!). for insane in -1e200, 1e200: self.assertRaises(ValueError, self.theclass.fromtimestamp, insane) def test_today(self): import time # We claim that today() is like fromtimestamp(time.time()), so # prove it. for dummy in range(3): today = self.theclass.today() ts = time.time() todayagain = self.theclass.fromtimestamp(ts) if today == todayagain: break # There are several legit reasons that could fail: # 1. It recently became midnight, between the today() and the # time() calls. # 2. The platform time() has such fine resolution that we'll # never get the same value twice. # 3. The platform time() has poor resolution, and we just # happened to call today() right before a resolution quantum # boundary. # 4. The system clock got fiddled between calls. # In any case, wait a little while and try again. time.sleep(0.1) # It worked or it didn't. If it didn't, assume it's reason #2, and # let the test pass if they're within half a second of each other. if today != todayagain: self.assertAlmostEqual(todayagain, today, delta=timedelta(seconds=0.5)) def test_weekday(self): for i in range(7): # March 4, 2002 is a Monday self.assertEqual(self.theclass(2002, 3, 4+i).weekday(), i) self.assertEqual(self.theclass(2002, 3, 4+i).isoweekday(), i+1) # January 2, 1956 is a Monday self.assertEqual(self.theclass(1956, 1, 2+i).weekday(), i) self.assertEqual(self.theclass(1956, 1, 2+i).isoweekday(), i+1) def test_isocalendar(self): # Check examples from # http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm for i in range(7): d = self.theclass(2003, 12, 22+i) self.assertEqual(d.isocalendar(), (2003, 52, i+1)) d = self.theclass(2003, 12, 29) + timedelta(i) self.assertEqual(d.isocalendar(), (2004, 1, i+1)) d = self.theclass(2004, 1, 5+i) self.assertEqual(d.isocalendar(), (2004, 2, i+1)) d = self.theclass(2009, 12, 21+i) self.assertEqual(d.isocalendar(), (2009, 52, i+1)) d = self.theclass(2009, 12, 28) + timedelta(i) self.assertEqual(d.isocalendar(), (2009, 53, i+1)) d = self.theclass(2010, 1, 4+i) self.assertEqual(d.isocalendar(), (2010, 1, i+1)) def test_iso_long_years(self): # Calculate long ISO years and compare to table from # http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm ISO_LONG_YEARS_TABLE = """ 4 32 60 88 9 37 65 93 15 43 71 99 20 48 76 26 54 82 105 133 161 189 111 139 167 195 116 144 172 122 150 178 128 156 184 201 229 257 285 207 235 263 291 212 240 268 296 218 246 274 224 252 280 303 331 359 387 308 336 364 392 314 342 370 398 320 348 376 325 353 381 """ iso_long_years = map(int, ISO_LONG_YEARS_TABLE.split()) iso_long_years.sort() L = [] for i in range(400): d = self.theclass(2000+i, 12, 31) d1 = self.theclass(1600+i, 12, 31) self.assertEqual(d.isocalendar()[1:], d1.isocalendar()[1:]) if d.isocalendar()[1] == 53: L.append(i) self.assertEqual(L, iso_long_years) def test_isoformat(self): t = self.theclass(2, 3, 2) self.assertEqual(t.isoformat(), "0002-03-02") def test_ctime(self): t = self.theclass(2002, 3, 2) self.assertEqual(t.ctime(), "Sat Mar 2 00:00:00 2002") @unittest.expectedFailure def test_strftime(self): t = self.theclass(2005, 3, 2) self.assertEqual(t.strftime("m:%m d:%d y:%y"), "m:03 d:02 y:05") self.assertEqual(t.strftime(""), "") # SF bug #761337 self.assertEqual(t.strftime('x'*1000), 'x'*1000) # SF bug #1556784 self.assertRaises(TypeError, t.strftime) # needs an arg self.assertRaises(TypeError, t.strftime, "one", "two") # too many args self.assertRaises(TypeError, t.strftime, 42) # arg wrong type # test that unicode input is allowed (issue 2782) self.assertEqual(t.strftime(u"%m"), "03") # A naive object replaces %z and %Z w/ empty strings. self.assertEqual(t.strftime("'%z' '%Z'"), "'' ''") #make sure that invalid format specifiers are handled correctly #self.assertRaises(ValueError, t.strftime, "%e") #self.assertRaises(ValueError, t.strftime, "%") #self.assertRaises(ValueError, t.strftime, "%#") #oh well, some systems just ignore those invalid ones. #at least, exercise them to make sure that no crashes #are generated for f in ["%e", "%", "%#"]: try: t.strftime(f) except ValueError: pass #check that this standard extension works t.strftime("%f") @unittest.expectedFailure def test_format(self): dt = self.theclass(2007, 9, 10) self.assertEqual(dt.__format__(''), str(dt)) # check that a derived class's __str__() gets called class A(self.theclass): def __str__(self): return 'A' a = A(2007, 9, 10) self.assertEqual(a.__format__(''), 'A') # check that a derived class's strftime gets called class B(self.theclass): def strftime(self, format_spec): return 'B' b = B(2007, 9, 10) self.assertEqual(b.__format__(''), str(dt)) for fmt in ["m:%m d:%d y:%y", "m:%m d:%d y:%y H:%H M:%M S:%S", "%z %Z", ]: self.assertEqual(dt.__format__(fmt), dt.strftime(fmt)) self.assertEqual(a.__format__(fmt), dt.strftime(fmt)) self.assertEqual(b.__format__(fmt), 'B') def test_resolution_info(self): self.assertIsInstance(self.theclass.min, self.theclass) self.assertIsInstance(self.theclass.max, self.theclass) self.assertIsInstance(self.theclass.resolution, timedelta) self.assertTrue(self.theclass.max > self.theclass.min) def test_extreme_timedelta(self): big = self.theclass.max - self.theclass.min # 3652058 days, 23 hours, 59 minutes, 59 seconds, 999999 microseconds n = (big.days*24*3600 + big.seconds)*1000000 + big.microseconds # n == 315537897599999999 ~= 2**58.13 justasbig = timedelta(0, 0, n) self.assertEqual(big, justasbig) self.assertEqual(self.theclass.min + big, self.theclass.max) self.assertEqual(self.theclass.max - big, self.theclass.min) def test_timetuple(self): for i in range(7): # January 2, 1956 is a Monday (0) d = self.theclass(1956, 1, 2+i) t = d.timetuple() self.assertEqual(t, (1956, 1, 2+i, 0, 0, 0, i, 2+i, -1)) # February 1, 1956 is a Wednesday (2) d = self.theclass(1956, 2, 1+i) t = d.timetuple() self.assertEqual(t, (1956, 2, 1+i, 0, 0, 0, (2+i)%7, 32+i, -1)) # March 1, 1956 is a Thursday (3), and is the 31+29+1 = 61st day # of the year. d = self.theclass(1956, 3, 1+i) t = d.timetuple() self.assertEqual(t, (1956, 3, 1+i, 0, 0, 0, (3+i)%7, 61+i, -1)) self.assertEqual(t.tm_year, 1956) self.assertEqual(t.tm_mon, 3) self.assertEqual(t.tm_mday, 1+i) self.assertEqual(t.tm_hour, 0) self.assertEqual(t.tm_min, 0) self.assertEqual(t.tm_sec, 0) self.assertEqual(t.tm_wday, (3+i)%7) self.assertEqual(t.tm_yday, 61+i) self.assertEqual(t.tm_isdst, -1) # def test_pickling(self): # args = 6, 7, 23 # orig = self.theclass(*args) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) def test_compare(self): t1 = self.theclass(2, 3, 4) t2 = self.theclass(2, 3, 4) self.assertTrue(t1 == t2) self.assertTrue(t1 <= t2) self.assertTrue(t1 >= t2) self.assertFalse(t1 != t2) self.assertFalse(t1 < t2) self.assertFalse(t1 > t2) self.assertEqual(cmp(t1, t2), 0) self.assertEqual(cmp(t2, t1), 0) for args in (3, 3, 3), (2, 4, 4), (2, 3, 5): t2 = self.theclass(*args) # this is larger than t1 self.assertTrue(t1 < t2) self.assertTrue(t2 > t1) self.assertTrue(t1 <= t2) self.assertTrue(t2 >= t1) self.assertTrue(t1 != t2) self.assertTrue(t2 != t1) self.assertFalse(t1 == t2) self.assertFalse(t2 == t1) self.assertFalse(t1 > t2) self.assertFalse(t2 < t1) self.assertFalse(t1 >= t2) self.assertFalse(t2 <= t1) self.assertEqual(cmp(t1, t2), -1) self.assertEqual(cmp(t2, t1), 1) for badarg in OTHERSTUFF: self.assertEqual(t1 == badarg, False) self.assertEqual(t1 != badarg, True) self.assertEqual(badarg == t1, False) self.assertEqual(badarg != t1, True) self.assertRaises(TypeError, lambda: t1 < badarg) self.assertRaises(TypeError, lambda: t1 > badarg) self.assertRaises(TypeError, lambda: t1 >= badarg) self.assertRaises(TypeError, lambda: badarg <= t1) self.assertRaises(TypeError, lambda: badarg < t1) self.assertRaises(TypeError, lambda: badarg > t1) self.assertRaises(TypeError, lambda: badarg >= t1) def test_mixed_compare(self): our = self.theclass(2000, 4, 5) self.assertRaises(TypeError, cmp, our, 1) self.assertRaises(TypeError, cmp, 1, our) class AnotherDateTimeClass(object): def __cmp__(self, other): # Return "equal" so calling this can't be confused with # compare-by-address (which never says "equal" for distinct # objects). return 0 __hash__ = None # Silence Py3k warning # This still errors, because date and datetime comparison raise # TypeError instead of NotImplemented when they don't know what to # do, in order to stop comparison from falling back to the default # compare-by-address. their = AnotherDateTimeClass() self.assertRaises(TypeError, cmp, our, their) # Oops: The next stab raises TypeError in the C implementation, # but not in the Python implementation of datetime. The difference # is due to that the Python implementation defines __cmp__ but # the C implementation defines tp_richcompare. This is more pain # to fix than it's worth, so commenting out the test. # self.assertEqual(cmp(their, our), 0) # But date and datetime comparison return NotImplemented instead if the # other object has a timetuple attr. This gives the other object a # chance to do the comparison. class Comparable(AnotherDateTimeClass): def timetuple(self): return () their = Comparable() self.assertEqual(cmp(our, their), 0) self.assertEqual(cmp(their, our), 0) self.assertTrue(our == their) self.assertTrue(their == our) def test_bool(self): # All dates are considered true. self.assertTrue(self.theclass.min) self.assertTrue(self.theclass.max) @unittest.expectedFailure def test_strftime_out_of_range(self): # For nasty technical reasons, we can't handle years before 1900. cls = self.theclass self.assertEqual(cls(1900, 1, 1).strftime("%Y"), "1900") for y in 1, 49, 51, 99, 100, 1000, 1899: self.assertRaises(ValueError, cls(y, 1, 1).strftime, "%Y") @unittest.expectedFailure def test_replace(self): cls = self.theclass args = [1, 2, 3] base = cls(*args) self.assertEqual(base, base.replace()) i = 0 for name, newval in (("year", 2), ("month", 3), ("day", 4)): newargs = args[:] newargs[i] = newval expected = cls(*newargs) got = base.replace(**{name: newval}) self.assertEqual(expected, got) i += 1 # Out of bounds. base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) @unittest.expectedFailure def test_subclass_date(self): class C(self.theclass): theAnswer = 42 def __new__(cls, *args, **kws): temp = kws.copy() extra = temp.pop('extra') result = self.theclass.__new__(cls, *args, **temp) result.extra = extra return result def newmeth(self, start): return start + self.year + self.month args = 2003, 4, 14 dt1 = self.theclass(*args) dt2 = C(*args, **{'extra': 7}) self.assertEqual(dt2.__class__, C) self.assertEqual(dt2.theAnswer, 42) self.assertEqual(dt2.extra, 7) self.assertEqual(dt1.toordinal(), dt2.toordinal()) self.assertEqual(dt2.newmeth(-7), dt1.year + dt1.month - 7) # def test_pickling_subclass_date(self): # args = 6, 7, 23 # orig = SubclassDate(*args) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) # def test_backdoor_resistance(self): # # For fast unpickling, the constructor accepts a pickle string. # # This is a low-overhead backdoor. A user can (by intent or # # mistake) pass a string directly, which (if it's the right length) # # will get treated like a pickle, and bypass the normal sanity # # checks in the constructor. This can create insane objects. # # The constructor doesn't want to burn the time to validate all # # fields, but does check the month field. This stops, e.g., # # datetime.datetime('1995-03-25') from yielding an insane object. # base = '1995-03-25' # if not issubclass(self.theclass, datetime): # base = base[:4] # for month_byte in '9', chr(0), chr(13), '\xff': # self.assertRaises(TypeError, self.theclass, # base[:2] + month_byte + base[3:]) # for ord_byte in range(1, 13): # # This shouldn't blow up because of the month byte alone. If # # the implementation changes to do more-careful checking, it may # # blow up because other fields are insane. # self.theclass(base[:2] + chr(ord_byte) + base[3:]) ############################################################################# # datetime tests class SubclassDatetime(datetime): sub_var = 1 class TestDateTime(TestDate): theclass = datetime def test_basic_attributes(self): dt = self.theclass(2002, 3, 1, 12, 0) self.assertEqual(dt.year, 2002) self.assertEqual(dt.month, 3) self.assertEqual(dt.day, 1) self.assertEqual(dt.hour, 12) self.assertEqual(dt.minute, 0) self.assertEqual(dt.second, 0) self.assertEqual(dt.microsecond, 0) def test_basic_attributes_nonzero(self): # Make sure all attributes are non-zero so bugs in # bit-shifting access show up. dt = self.theclass(2002, 3, 1, 12, 59, 59, 8000) self.assertEqual(dt.year, 2002) self.assertEqual(dt.month, 3) self.assertEqual(dt.day, 1) self.assertEqual(dt.hour, 12) self.assertEqual(dt.minute, 59) self.assertEqual(dt.second, 59) self.assertEqual(dt.microsecond, 8000) @unittest.expectedFailure def test_roundtrip(self): for dt in (self.theclass(1, 2, 3, 4, 5, 6, 7), self.theclass.now()): # Verify dt -> string -> datetime identity. s = repr(dt) self.assertTrue(s.startswith('datetime.')) s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) # Verify identity via reconstructing from pieces. dt2 = self.theclass(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.microsecond) self.assertEqual(dt, dt2) @unittest.expectedFailure def test_isoformat(self): t = self.theclass(2, 3, 2, 4, 5, 1, 123) self.assertEqual(t.isoformat(), "0002-03-02T04:05:01.000123") self.assertEqual(t.isoformat('T'), "0002-03-02T04:05:01.000123") self.assertEqual(t.isoformat(' '), "0002-03-02 04:05:01.000123") self.assertEqual(t.isoformat('\x00'), "0002-03-02\x0004:05:01.000123") # str is ISO format with the separator forced to a blank. self.assertEqual(str(t), "0002-03-02 04:05:01.000123") t = self.theclass(2, 3, 2) self.assertEqual(t.isoformat(), "0002-03-02T00:00:00") self.assertEqual(t.isoformat('T'), "0002-03-02T00:00:00") self.assertEqual(t.isoformat(' '), "0002-03-02 00:00:00") # str is ISO format with the separator forced to a blank. self.assertEqual(str(t), "0002-03-02 00:00:00") @unittest.expectedFailure def test_format(self): dt = self.theclass(2007, 9, 10, 4, 5, 1, 123) self.assertEqual(dt.__format__(''), str(dt)) # check that a derived class's __str__() gets called class A(self.theclass): def __str__(self): return 'A' a = A(2007, 9, 10, 4, 5, 1, 123) self.assertEqual(a.__format__(''), 'A') # check that a derived class's strftime gets called class B(self.theclass): def strftime(self, format_spec): return 'B' b = B(2007, 9, 10, 4, 5, 1, 123) self.assertEqual(b.__format__(''), str(dt)) for fmt in ["m:%m d:%d y:%y", "m:%m d:%d y:%y H:%H M:%M S:%S", "%z %Z", ]: self.assertEqual(dt.__format__(fmt), dt.strftime(fmt)) self.assertEqual(a.__format__(fmt), dt.strftime(fmt)) self.assertEqual(b.__format__(fmt), 'B') @unittest.expectedFailure def test_more_ctime(self): # Test fields that TestDate doesn't touch. import time t = self.theclass(2002, 3, 2, 18, 3, 5, 123) self.assertEqual(t.ctime(), "Sat Mar 2 18:03:05 2002") # Oops! The next line fails on Win2K under MSVC 6, so it's commented # out. The difference is that t.ctime() produces " 2" for the day, # but platform ctime() produces "02" for the day. According to # C99, t.ctime() is correct here. # self.assertEqual(t.ctime(), time.ctime(time.mktime(t.timetuple()))) # So test a case where that difference doesn't matter. t = self.theclass(2002, 3, 22, 18, 3, 5, 123) self.assertEqual(t.ctime(), time.ctime(time.mktime(t.timetuple()))) def test_tz_independent_comparing(self): dt1 = self.theclass(2002, 3, 1, 9, 0, 0) dt2 = self.theclass(2002, 3, 1, 10, 0, 0) dt3 = self.theclass(2002, 3, 1, 9, 0, 0) self.assertEqual(dt1, dt3) self.assertTrue(dt2 > dt3) # Make sure comparison doesn't forget microseconds, and isn't done # via comparing a float timestamp (an IEEE double doesn't have enough # precision to span microsecond resolution across years 1 thru 9999, # so comparing via timestamp necessarily calls some distinct values # equal). dt1 = self.theclass(MAXYEAR, 12, 31, 23, 59, 59, 999998) us = timedelta(microseconds=1) dt2 = dt1 + us self.assertEqual(dt2 - dt1, us) self.assertTrue(dt1 < dt2) @unittest.expectedFailure def test_strftime_with_bad_tzname_replace(self): # verify ok if tzinfo.tzname().replace() returns a non-string class MyTzInfo(FixedOffset): def tzname(self, dt): class MyStr(str): def replace(self, *args): return None return MyStr('name') t = self.theclass(2005, 3, 2, 0, 0, 0, 0, MyTzInfo(3, 'name')) self.assertRaises(TypeError, t.strftime, '%Z') def test_bad_constructor_arguments(self): # bad years self.theclass(MINYEAR, 1, 1) # no exception self.theclass(MAXYEAR, 1, 1) # no exception self.assertRaises(ValueError, self.theclass, MINYEAR-1, 1, 1) self.assertRaises(ValueError, self.theclass, MAXYEAR+1, 1, 1) # bad months self.theclass(2000, 1, 1) # no exception self.theclass(2000, 12, 1) # no exception self.assertRaises(ValueError, self.theclass, 2000, 0, 1) self.assertRaises(ValueError, self.theclass, 2000, 13, 1) # bad days self.theclass(2000, 2, 29) # no exception self.theclass(2004, 2, 29) # no exception self.theclass(2400, 2, 29) # no exception self.assertRaises(ValueError, self.theclass, 2000, 2, 30) self.assertRaises(ValueError, self.theclass, 2001, 2, 29) self.assertRaises(ValueError, self.theclass, 2100, 2, 29) self.assertRaises(ValueError, self.theclass, 1900, 2, 29) self.assertRaises(ValueError, self.theclass, 2000, 1, 0) self.assertRaises(ValueError, self.theclass, 2000, 1, 32) # bad hours self.theclass(2000, 1, 31, 0) # no exception self.theclass(2000, 1, 31, 23) # no exception self.assertRaises(ValueError, self.theclass, 2000, 1, 31, -1) self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 24) # bad minutes self.theclass(2000, 1, 31, 23, 0) # no exception self.theclass(2000, 1, 31, 23, 59) # no exception self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, -1) self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, 60) # bad seconds self.theclass(2000, 1, 31, 23, 59, 0) # no exception self.theclass(2000, 1, 31, 23, 59, 59) # no exception self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, 59, -1) self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, 59, 60) # bad microseconds self.theclass(2000, 1, 31, 23, 59, 59, 0) # no exception self.theclass(2000, 1, 31, 23, 59, 59, 999999) # no exception self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, 59, 59, -1) self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, 59, 59, 1000000) def test_hash_equality(self): d = self.theclass(2000, 12, 31, 23, 30, 17) e = self.theclass(2000, 12, 31, 23, 30, 17) self.assertEqual(d, e) self.assertEqual(hash(d), hash(e)) dic = {d: 1} dic[e] = 2 self.assertEqual(len(dic), 1) self.assertEqual(dic[d], 2) self.assertEqual(dic[e], 2) d = self.theclass(2001, 1, 1, 0, 5, 17) e = self.theclass(2001, 1, 1, 0, 5, 17) self.assertEqual(d, e) self.assertEqual(hash(d), hash(e)) dic = {d: 1} dic[e] = 2 self.assertEqual(len(dic), 1) self.assertEqual(dic[d], 2) self.assertEqual(dic[e], 2) def test_computations(self): a = self.theclass(2002, 1, 31) b = self.theclass(1956, 1, 31) diff = a-b self.assertEqual(diff.days, 46*365 + len(range(1956, 2002, 4))) self.assertEqual(diff.seconds, 0) self.assertEqual(diff.microseconds, 0) a = self.theclass(2002, 3, 2, 17, 6) millisec = timedelta(0, 0, 1000) hour = timedelta(0, 3600) day = timedelta(1) week = timedelta(7) self.assertEqual(a + hour, self.theclass(2002, 3, 2, 18, 6)) self.assertEqual(hour + a, self.theclass(2002, 3, 2, 18, 6)) self.assertEqual(a + 10*hour, self.theclass(2002, 3, 3, 3, 6)) self.assertEqual(a - hour, self.theclass(2002, 3, 2, 16, 6)) self.assertEqual(-hour + a, self.theclass(2002, 3, 2, 16, 6)) self.assertEqual(a - hour, a + -hour) self.assertEqual(a - 20*hour, self.theclass(2002, 3, 1, 21, 6)) self.assertEqual(a + day, self.theclass(2002, 3, 3, 17, 6)) self.assertEqual(a - day, self.theclass(2002, 3, 1, 17, 6)) self.assertEqual(a + week, self.theclass(2002, 3, 9, 17, 6)) self.assertEqual(a - week, self.theclass(2002, 2, 23, 17, 6)) self.assertEqual(a + 52*week, self.theclass(2003, 3, 1, 17, 6)) self.assertEqual(a - 52*week, self.theclass(2001, 3, 3, 17, 6)) self.assertEqual((a + week) - a, week) self.assertEqual((a + day) - a, day) self.assertEqual((a + hour) - a, hour) self.assertEqual((a + millisec) - a, millisec) self.assertEqual((a - week) - a, -week) self.assertEqual((a - day) - a, -day) self.assertEqual((a - hour) - a, -hour) self.assertEqual((a - millisec) - a, -millisec) self.assertEqual(a - (a + week), -week) self.assertEqual(a - (a + day), -day) self.assertEqual(a - (a + hour), -hour) self.assertEqual(a - (a + millisec), -millisec) self.assertEqual(a - (a - week), week) self.assertEqual(a - (a - day), day) self.assertEqual(a - (a - hour), hour) self.assertEqual(a - (a - millisec), millisec) self.assertEqual(a + (week + day + hour + millisec), self.theclass(2002, 3, 10, 18, 6, 0, 1000)) self.assertEqual(a + (week + day + hour + millisec), (((a + week) + day) + hour) + millisec) self.assertEqual(a - (week + day + hour + millisec), self.theclass(2002, 2, 22, 16, 5, 59, 999000)) self.assertEqual(a - (week + day + hour + millisec), (((a - week) - day) - hour) - millisec) # Add/sub ints, longs, floats should be illegal for i in 1, 1L, 1.0: self.assertRaises(TypeError, lambda: a+i) self.assertRaises(TypeError, lambda: a-i) self.assertRaises(TypeError, lambda: i+a) self.assertRaises(TypeError, lambda: i-a) # delta - datetime is senseless. self.assertRaises(TypeError, lambda: day - a) # mixing datetime and (delta or datetime) via * or // is senseless self.assertRaises(TypeError, lambda: day * a) self.assertRaises(TypeError, lambda: a * day) self.assertRaises(TypeError, lambda: day // a) self.assertRaises(TypeError, lambda: a // day) self.assertRaises(TypeError, lambda: a * a) self.assertRaises(TypeError, lambda: a // a) # datetime + datetime is senseless self.assertRaises(TypeError, lambda: a + a) # def test_pickling(self): # args = 6, 7, 23, 20, 59, 1, 64**2 # orig = self.theclass(*args) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) # def test_more_pickling(self): # a = self.theclass(2003, 2, 7, 16, 48, 37, 444116) # for proto in range(pickle.HIGHEST_PROTOCOL + 1): # s = pickle.dumps(a, proto) # b = pickle.loads(s) # self.assertEqual(b.year, 2003) # self.assertEqual(b.month, 2) # self.assertEqual(b.day, 7) # def test_pickling_subclass_datetime(self): # args = 6, 7, 23, 20, 59, 1, 64**2 # orig = SubclassDatetime(*args) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) def test_more_compare(self): # The test_compare() inherited from TestDate covers the error cases. # We just want to test lexicographic ordering on the members datetime # has that date lacks. args = [2000, 11, 29, 20, 58, 16, 999998] t1 = self.theclass(*args) t2 = self.theclass(*args) self.assertTrue(t1 == t2) self.assertTrue(t1 <= t2) self.assertTrue(t1 >= t2) self.assertFalse(t1 != t2) self.assertFalse(t1 < t2) self.assertFalse(t1 > t2) self.assertEqual(cmp(t1, t2), 0) self.assertEqual(cmp(t2, t1), 0) for i in range(len(args)): newargs = args[:] newargs[i] = args[i] + 1 t2 = self.theclass(*newargs) # this is larger than t1 self.assertTrue(t1 < t2) self.assertTrue(t2 > t1) self.assertTrue(t1 <= t2) self.assertTrue(t2 >= t1) self.assertTrue(t1 != t2) self.assertTrue(t2 != t1) self.assertFalse(t1 == t2) self.assertFalse(t2 == t1) self.assertFalse(t1 > t2) self.assertFalse(t2 < t1) self.assertFalse(t1 >= t2) self.assertFalse(t2 <= t1) self.assertEqual(cmp(t1, t2), -1) self.assertEqual(cmp(t2, t1), 1) # A helper for timestamp constructor tests. def verify_field_equality(self, expected, got): self.assertEqual(expected.tm_year, got.year) self.assertEqual(expected.tm_mon, got.month) self.assertEqual(expected.tm_mday, got.day) self.assertEqual(expected.tm_hour, got.hour) self.assertEqual(expected.tm_min, got.minute) self.assertEqual(expected.tm_sec, got.second) def test_fromtimestamp(self): import time ts = time.time() expected = time.localtime(ts) got = self.theclass.fromtimestamp(ts) self.verify_field_equality(expected, got) def test_utcfromtimestamp(self): import time ts = time.time() expected = time.gmtime(ts) got = self.theclass.utcfromtimestamp(ts) self.verify_field_equality(expected, got) def test_microsecond_rounding(self): # Test whether fromtimestamp "rounds up" floats that are less # than one microsecond smaller than an integer. self.assertEqual(self.theclass.fromtimestamp(0.9999999), self.theclass.fromtimestamp(1)) @unittest.expectedFailure def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, # and that this test will fail there. This test should # exempt such platforms (provided they return reasonable # results!). for insane in -1e200, 1e200: self.assertRaises(ValueError, self.theclass.fromtimestamp, insane) @unittest.expectedFailure def test_insane_utcfromtimestamp(self): # It's possible that some platform maps time_t to double, # and that this test will fail there. This test should # exempt such platforms (provided they return reasonable # results!). for insane in -1e200, 1e200: self.assertRaises(ValueError, self.theclass.utcfromtimestamp, insane) # @unittest.skipIf(sys.platform == "win32", "Windows doesn't accept negative timestamps") # def test_negative_float_fromtimestamp(self): # # The result is tz-dependent; at least test that this doesn't # # fail (like it did before bug 1646728 was fixed). # self.theclass.fromtimestamp(-1.05) # @unittest.skipIf(sys.platform == "win32", "Windows doesn't accept negative timestamps") # def test_negative_float_utcfromtimestamp(self): # d = self.theclass.utcfromtimestamp(-1.05) # self.assertEqual(d, self.theclass(1969, 12, 31, 23, 59, 58, 950000)) def test_utcnow(self): import time # Call it a success if utcnow() and utcfromtimestamp() are within # a second of each other. tolerance = timedelta(seconds=1) for dummy in range(3): from_now = self.theclass.utcnow() from_timestamp = self.theclass.utcfromtimestamp(time.time()) if abs(from_timestamp - from_now) <= tolerance: break # Else try again a few times. self.assertLessEqual(abs(from_timestamp - from_now), tolerance) # def test_strptime(self): # import _strptime # string = '2004-12-01 13:02:47.197' # format = '%Y-%m-%d %H:%M:%S.%f' # result, frac = _strptime._strptime(string, format) # expected = self.theclass(*(result[0:6]+(frac,))) # got = self.theclass.strptime(string, format) # self.assertEqual(expected, got) def test_more_timetuple(self): # This tests fields beyond those tested by the TestDate.test_timetuple. t = self.theclass(2004, 12, 31, 6, 22, 33) self.assertEqual(t.timetuple(), (2004, 12, 31, 6, 22, 33, 4, 366, -1)) self.assertEqual(t.timetuple(), (t.year, t.month, t.day, t.hour, t.minute, t.second, t.weekday(), t.toordinal() - date(t.year, 1, 1).toordinal() + 1, -1)) tt = t.timetuple() self.assertEqual(tt.tm_year, t.year) self.assertEqual(tt.tm_mon, t.month) self.assertEqual(tt.tm_mday, t.day) self.assertEqual(tt.tm_hour, t.hour) self.assertEqual(tt.tm_min, t.minute) self.assertEqual(tt.tm_sec, t.second) self.assertEqual(tt.tm_wday, t.weekday()) self.assertEqual(tt.tm_yday, t.toordinal() - date(t.year, 1, 1).toordinal() + 1) self.assertEqual(tt.tm_isdst, -1) @unittest.expectedFailure def test_more_strftime(self): # This tests fields beyond those tested by the TestDate.test_strftime. t = self.theclass(2004, 12, 31, 6, 22, 33, 47) self.assertEqual(t.strftime("%m %d %y %f %S %M %H %j"), "12 31 04 000047 33 22 06 366") def test_extract(self): dt = self.theclass(2002, 3, 4, 18, 45, 3, 1234) self.assertEqual(dt.date(), date(2002, 3, 4)) self.assertEqual(dt.time(), time(18, 45, 3, 1234)) def test_combine(self): d = date(2002, 3, 4) t = time(18, 45, 3, 1234) expected = self.theclass(2002, 3, 4, 18, 45, 3, 1234) combine = self.theclass.combine dt = combine(d, t) self.assertEqual(dt, expected) dt = combine(time=t, date=d) self.assertEqual(dt, expected) self.assertEqual(d, dt.date()) self.assertEqual(t, dt.time()) self.assertEqual(dt, combine(dt.date(), dt.time())) self.assertRaises(TypeError, combine) # need an arg self.assertRaises(TypeError, combine, d) # need two args self.assertRaises(TypeError, combine, t, d) # args reversed self.assertRaises(TypeError, combine, d, t, 1) # too many args self.assertRaises(TypeError, combine, "date", "time") # wrong types @unittest.expectedFailure def test_replace(self): cls = self.theclass args = [1, 2, 3, 4, 5, 6, 7] base = cls(*args) self.assertEqual(base, base.replace()) i = 0 for name, newval in (("year", 2), ("month", 3), ("day", 4), ("hour", 5), ("minute", 6), ("second", 7), ("microsecond", 8)): newargs = args[:] newargs[i] = newval expected = cls(*newargs) got = base.replace(**{name: newval}) self.assertEqual(expected, got) i += 1 # Out of bounds. base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) def test_astimezone(self): # Pretty boring! The TZ test is more interesting here. astimezone() # simply can't be applied to a naive object. dt = self.theclass.now() f = FixedOffset(44, "") self.assertRaises(TypeError, dt.astimezone) # not enough args self.assertRaises(TypeError, dt.astimezone, f, f) # too many args self.assertRaises(TypeError, dt.astimezone, dt) # arg wrong type self.assertRaises(ValueError, dt.astimezone, f) # naive self.assertRaises(ValueError, dt.astimezone, tz=f) # naive class Bogus(tzinfo): def utcoffset(self, dt): return None def dst(self, dt): return timedelta(0) bog = Bogus() self.assertRaises(ValueError, dt.astimezone, bog) # naive class AlsoBogus(tzinfo): def utcoffset(self, dt): return timedelta(0) def dst(self, dt): return None alsobog = AlsoBogus() self.assertRaises(ValueError, dt.astimezone, alsobog) # also naive @unittest.expectedFailure def test_subclass_datetime(self): class C(self.theclass): theAnswer = 42 def __new__(cls, *args, **kws): temp = kws.copy() extra = temp.pop('extra') result = self.theclass.__new__(cls, *args, **temp) result.extra = extra return result def newmeth(self, start): return start + self.year + self.month + self.second args = 2003, 4, 14, 12, 13, 41 dt1 = self.theclass(*args) dt2 = C(*args, **{'extra': 7}) self.assertEqual(dt2.__class__, C) self.assertEqual(dt2.theAnswer, 42) self.assertEqual(dt2.extra, 7) self.assertEqual(dt1.toordinal(), dt2.toordinal()) self.assertEqual(dt2.newmeth(-7), dt1.year + dt1.month + dt1.second - 7) class SubclassTime(time): sub_var = 1 class TestTime(HarmlessMixedComparison, unittest.TestCase): theclass = time def test_basic_attributes(self): t = self.theclass(12, 0) self.assertEqual(t.hour, 12) self.assertEqual(t.minute, 0) self.assertEqual(t.second, 0) self.assertEqual(t.microsecond, 0) def test_basic_attributes_nonzero(self): # Make sure all attributes are non-zero so bugs in # bit-shifting access show up. t = self.theclass(12, 59, 59, 8000) self.assertEqual(t.hour, 12) self.assertEqual(t.minute, 59) self.assertEqual(t.second, 59) self.assertEqual(t.microsecond, 8000) @unittest.expectedFailure def test_roundtrip(self): t = self.theclass(1, 2, 3, 4) # Verify t -> string -> time identity. s = repr(t) self.assertTrue(s.startswith('datetime.')) s = s[9:] t2 = eval(s) self.assertEqual(t, t2) # Verify identity via reconstructing from pieces. t2 = self.theclass(t.hour, t.minute, t.second, t.microsecond) self.assertEqual(t, t2) def test_comparing(self): args = [1, 2, 3, 4] t1 = self.theclass(*args) t2 = self.theclass(*args) self.assertTrue(t1 == t2) self.assertTrue(t1 <= t2) self.assertTrue(t1 >= t2) self.assertFalse(t1 != t2) self.assertFalse(t1 < t2) self.assertFalse(t1 > t2) self.assertEqual(cmp(t1, t2), 0) self.assertEqual(cmp(t2, t1), 0) for i in range(len(args)): newargs = args[:] newargs[i] = args[i] + 1 t2 = self.theclass(*newargs) # this is larger than t1 self.assertTrue(t1 < t2) self.assertTrue(t2 > t1) self.assertTrue(t1 <= t2) self.assertTrue(t2 >= t1) self.assertTrue(t1 != t2) self.assertTrue(t2 != t1) self.assertFalse(t1 == t2) self.assertFalse(t2 == t1) self.assertFalse(t1 > t2) self.assertFalse(t2 < t1) self.assertFalse(t1 >= t2) self.assertFalse(t2 <= t1) self.assertEqual(cmp(t1, t2), -1) self.assertEqual(cmp(t2, t1), 1) for badarg in OTHERSTUFF: self.assertEqual(t1 == badarg, False) self.assertEqual(t1 != badarg, True) self.assertEqual(badarg == t1, False) self.assertEqual(badarg != t1, True) self.assertRaises(TypeError, lambda: t1 <= badarg) self.assertRaises(TypeError, lambda: t1 < badarg) self.assertRaises(TypeError, lambda: t1 > badarg) self.assertRaises(TypeError, lambda: t1 >= badarg) self.assertRaises(TypeError, lambda: badarg <= t1) self.assertRaises(TypeError, lambda: badarg < t1) self.assertRaises(TypeError, lambda: badarg > t1) self.assertRaises(TypeError, lambda: badarg >= t1) def test_bad_constructor_arguments(self): # bad hours self.theclass(0, 0) # no exception self.theclass(23, 0) # no exception self.assertRaises(ValueError, self.theclass, -1, 0) self.assertRaises(ValueError, self.theclass, 24, 0) # bad minutes self.theclass(23, 0) # no exception self.theclass(23, 59) # no exception self.assertRaises(ValueError, self.theclass, 23, -1) self.assertRaises(ValueError, self.theclass, 23, 60) # bad seconds self.theclass(23, 59, 0) # no exception self.theclass(23, 59, 59) # no exception self.assertRaises(ValueError, self.theclass, 23, 59, -1) self.assertRaises(ValueError, self.theclass, 23, 59, 60) # bad microseconds self.theclass(23, 59, 59, 0) # no exception self.theclass(23, 59, 59, 999999) # no exception self.assertRaises(ValueError, self.theclass, 23, 59, 59, -1) self.assertRaises(ValueError, self.theclass, 23, 59, 59, 1000000) def test_hash_equality(self): d = self.theclass(23, 30, 17) e = self.theclass(23, 30, 17) self.assertEqual(d, e) self.assertEqual(hash(d), hash(e)) dic = {d: 1} dic[e] = 2 self.assertEqual(len(dic), 1) self.assertEqual(dic[d], 2) self.assertEqual(dic[e], 2) d = self.theclass(0, 5, 17) e = self.theclass(0, 5, 17) self.assertEqual(d, e) self.assertEqual(hash(d), hash(e)) dic = {d: 1} dic[e] = 2 self.assertEqual(len(dic), 1) self.assertEqual(dic[d], 2) self.assertEqual(dic[e], 2) def test_isoformat(self): t = self.theclass(4, 5, 1, 123) self.assertEqual(t.isoformat(), "04:05:01.000123") self.assertEqual(t.isoformat(), str(t)) t = self.theclass() self.assertEqual(t.isoformat(), "00:00:00") self.assertEqual(t.isoformat(), str(t)) t = self.theclass(microsecond=1) self.assertEqual(t.isoformat(), "00:00:00.000001") self.assertEqual(t.isoformat(), str(t)) t = self.theclass(microsecond=10) self.assertEqual(t.isoformat(), "00:00:00.000010") self.assertEqual(t.isoformat(), str(t)) t = self.theclass(microsecond=100) self.assertEqual(t.isoformat(), "00:00:00.000100") self.assertEqual(t.isoformat(), str(t)) t = self.theclass(microsecond=1000) self.assertEqual(t.isoformat(), "00:00:00.001000") self.assertEqual(t.isoformat(), str(t)) t = self.theclass(microsecond=10000) self.assertEqual(t.isoformat(), "00:00:00.010000") self.assertEqual(t.isoformat(), str(t)) t = self.theclass(microsecond=100000) self.assertEqual(t.isoformat(), "00:00:00.100000") self.assertEqual(t.isoformat(), str(t)) def test_1653736(self): # verify it doesn't accept extra keyword arguments t = self.theclass(second=1) self.assertRaises(TypeError, t.isoformat, foo=3) @unittest.expectedFailure def test_strftime(self): t = self.theclass(1, 2, 3, 4) self.assertEqual(t.strftime('%H %M %S %f'), "01 02 03 000004") # A naive object replaces %z and %Z with empty strings. self.assertEqual(t.strftime("'%z' '%Z'"), "'' ''") @unittest.expectedFailure def test_format(self): t = self.theclass(1, 2, 3, 4) self.assertEqual(t.__format__(''), str(t)) # check that a derived class's __str__() gets called class A(self.theclass): def __str__(self): return 'A' a = A(1, 2, 3, 4) self.assertEqual(a.__format__(''), 'A') # check that a derived class's strftime gets called class B(self.theclass): def strftime(self, format_spec): return 'B' b = B(1, 2, 3, 4) self.assertEqual(b.__format__(''), str(t)) for fmt in ['%H %M %S', ]: self.assertEqual(t.__format__(fmt), t.strftime(fmt)) self.assertEqual(a.__format__(fmt), t.strftime(fmt)) self.assertEqual(b.__format__(fmt), 'B') def test_str(self): self.assertEqual(str(self.theclass(1, 2, 3, 4)), "01:02:03.000004") self.assertEqual(str(self.theclass(10, 2, 3, 4000)), "10:02:03.004000") self.assertEqual(str(self.theclass(0, 2, 3, 400000)), "00:02:03.400000") self.assertEqual(str(self.theclass(12, 2, 3, 0)), "12:02:03") self.assertEqual(str(self.theclass(23, 15, 0, 0)), "23:15:00") def test_repr(self): name = 'datetime.' + self.theclass.__name__ self.assertEqual(repr(self.theclass(1, 2, 3, 4)), "%s(1, 2, 3, 4)" % name) self.assertEqual(repr(self.theclass(10, 2, 3, 4000)), "%s(10, 2, 3, 4000)" % name) self.assertEqual(repr(self.theclass(0, 2, 3, 400000)), "%s(0, 2, 3, 400000)" % name) self.assertEqual(repr(self.theclass(12, 2, 3, 0)), "%s(12, 2, 3)" % name) self.assertEqual(repr(self.theclass(23, 15, 0, 0)), "%s(23, 15)" % name) def test_resolution_info(self): self.assertIsInstance(self.theclass.min, self.theclass) self.assertIsInstance(self.theclass.max, self.theclass) self.assertIsInstance(self.theclass.resolution, timedelta) self.assertTrue(self.theclass.max > self.theclass.min) # def test_pickling(self): # args = 20, 59, 16, 64**2 # orig = self.theclass(*args) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) # def test_pickling_subclass_time(self): # args = 20, 59, 16, 64**2 # orig = SubclassTime(*args) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) def test_bool(self): cls = self.theclass self.assertTrue(cls(1)) self.assertTrue(cls(0, 1)) self.assertTrue(cls(0, 0, 1)) self.assertTrue(cls(0, 0, 0, 1)) self.assertFalse(cls(0)) self.assertFalse(cls()) @unittest.expectedFailure def test_replace(self): cls = self.theclass args = [1, 2, 3, 4] base = cls(*args) self.assertEqual(base, base.replace()) i = 0 for name, newval in (("hour", 5), ("minute", 6), ("second", 7), ("microsecond", 8)): newargs = args[:] newargs[i] = newval expected = cls(*newargs) got = base.replace(**{name: newval}) self.assertEqual(expected, got) i += 1 # Out of bounds. base = cls(1) self.assertRaises(ValueError, base.replace, hour=24) self.assertRaises(ValueError, base.replace, minute=-1) self.assertRaises(ValueError, base.replace, second=100) self.assertRaises(ValueError, base.replace, microsecond=1000000) @unittest.expectedFailure def test_subclass_time(self): class C(self.theclass): theAnswer = 42 def __new__(cls, *args, **kws): temp = kws.copy() extra = temp.pop('extra') result = self.theclass.__new__(cls, *args, **temp) result.extra = extra return result def newmeth(self, start): return start + self.hour + self.second args = 4, 5, 6 dt1 = self.theclass(*args) dt2 = C(*args, **{'extra': 7}) self.assertEqual(dt2.__class__, C) self.assertEqual(dt2.theAnswer, 42) self.assertEqual(dt2.extra, 7) self.assertEqual(dt1.isoformat(), dt2.isoformat()) self.assertEqual(dt2.newmeth(-7), dt1.hour + dt1.second - 7) def test_backdoor_resistance(self): # see TestDate.test_backdoor_resistance(). base = '2:59.0' for hour_byte in ' ', '9', chr(24), '\xff': self.assertRaises(TypeError, self.theclass, hour_byte + base[1:]) # A mixin for classes with a tzinfo= argument. Subclasses must define # theclass as a class attribute, and theclass(1, 1, 1, tzinfo=whatever) # must be legit (which is true for time and datetime). class TZInfoBase(object): def test_argument_passing(self): cls = self.theclass # A datetime passes itself on, a time passes None. class introspective(tzinfo): def tzname(self, dt): return dt and "real" or "none" def utcoffset(self, dt): return timedelta(minutes = dt and 42 or -42) dst = utcoffset obj = cls(1, 2, 3, tzinfo=introspective()) expected = cls is time and "none" or "real" self.assertEqual(obj.tzname(), expected) expected = timedelta(minutes=(cls is time and -42 or 42)) self.assertEqual(obj.utcoffset(), expected) self.assertEqual(obj.dst(), expected) def test_bad_tzinfo_classes(self): cls = self.theclass self.assertRaises(TypeError, cls, 1, 1, 1, tzinfo=12) class NiceTry(object): def __init__(self): pass def utcoffset(self, dt): pass self.assertRaises(TypeError, cls, 1, 1, 1, tzinfo=NiceTry) class BetterTry(tzinfo): def __init__(self): pass def utcoffset(self, dt): pass b = BetterTry() t = cls(1, 1, 1, tzinfo=b) self.assertIs(t.tzinfo, b) @unittest.skip('grumpy') def test_utc_offset_out_of_bounds(self): class Edgy(tzinfo): def __init__(self, offset): self.offset = timedelta(minutes=offset) def utcoffset(self, dt): return self.offset cls = self.theclass for offset, legit in ((-1440, False), (-1439, True), (1439, True), (1440, False)): if cls is time: t = cls(1, 2, 3, tzinfo=Edgy(offset)) elif cls is datetime: t = cls(6, 6, 6, 1, 2, 3, tzinfo=Edgy(offset)) else: assert 0, "impossible" if legit: aofs = abs(offset) h, m = divmod(aofs, 60) tag = "%c%02d:%02d" % (offset < 0 and '-' or '+', h, m) if isinstance(t, datetime): t = t.timetz() self.assertEqual(str(t), "01:02:03" + tag) else: self.assertRaises(ValueError, str, t) def test_tzinfo_classes(self): cls = self.theclass class C1(tzinfo): def utcoffset(self, dt): return None def dst(self, dt): return None def tzname(self, dt): return None for t in (cls(1, 1, 1), cls(1, 1, 1, tzinfo=None), cls(1, 1, 1, tzinfo=C1())): self.assertIsNone(t.utcoffset()) self.assertIsNone(t.dst()) self.assertIsNone(t.tzname()) class C3(tzinfo): def utcoffset(self, dt): return timedelta(minutes=-1439) def dst(self, dt): return timedelta(minutes=1439) def tzname(self, dt): return "aname" t = cls(1, 1, 1, tzinfo=C3()) self.assertEqual(t.utcoffset(), timedelta(minutes=-1439)) self.assertEqual(t.dst(), timedelta(minutes=1439)) self.assertEqual(t.tzname(), "aname") # Wrong types. class C4(tzinfo): def utcoffset(self, dt): return "aname" def dst(self, dt): return 7 def tzname(self, dt): return 0 t = cls(1, 1, 1, tzinfo=C4()) self.assertRaises(TypeError, t.utcoffset) self.assertRaises(TypeError, t.dst) self.assertRaises(TypeError, t.tzname) # Offset out of range. class C6(tzinfo): def utcoffset(self, dt): return timedelta(hours=-24) def dst(self, dt): return timedelta(hours=24) t = cls(1, 1, 1, tzinfo=C6()) self.assertRaises(ValueError, t.utcoffset) self.assertRaises(ValueError, t.dst) # Not a whole number of minutes. class C7(tzinfo): def utcoffset(self, dt): return timedelta(seconds=61) def dst(self, dt): return timedelta(microseconds=-81) t = cls(1, 1, 1, tzinfo=C7()) self.assertRaises(ValueError, t.utcoffset) self.assertRaises(ValueError, t.dst) @unittest.skip('grumpy') def test_aware_compare(self): cls = self.theclass # Ensure that utcoffset() gets ignored if the comparands have # the same tzinfo member. class OperandDependentOffset(tzinfo): def utcoffset(self, t): if t.minute < 10: # d0 and d1 equal after adjustment return timedelta(minutes=t.minute) else: # d2 off in the weeds return timedelta(minutes=59) base = cls(8, 9, 10, tzinfo=OperandDependentOffset()) d0 = base.replace(minute=3) d1 = base.replace(minute=9) d2 = base.replace(minute=11) for x in d0, d1, d2: for y in d0, d1, d2: got = cmp(x, y) expected = cmp(x.minute, y.minute) self.assertEqual(got, expected) # However, if they're different members, uctoffset is not ignored. # Note that a time can't actually have an operand-depedent offset, # though (and time.utcoffset() passes None to tzinfo.utcoffset()), # so skip this test for time. if cls is not time: d0 = base.replace(minute=3, tzinfo=OperandDependentOffset()) d1 = base.replace(minute=9, tzinfo=OperandDependentOffset()) d2 = base.replace(minute=11, tzinfo=OperandDependentOffset()) for x in d0, d1, d2: for y in d0, d1, d2: got = cmp(x, y) if (x is d0 or x is d1) and (y is d0 or y is d1): expected = 0 elif x is y is d2: expected = 0 elif x is d2: expected = -1 else: assert y is d2 expected = 1 self.assertEqual(got, expected) # Testing time objects with a non-None tzinfo. class TestTimeTZ(TestTime, TZInfoBase, unittest.TestCase): theclass = time def test_empty(self): t = self.theclass() self.assertEqual(t.hour, 0) self.assertEqual(t.minute, 0) self.assertEqual(t.second, 0) self.assertEqual(t.microsecond, 0) self.assertIsNone(t.tzinfo) @unittest.expectedFailure def test_zones(self): est = FixedOffset(-300, "EST", 1) utc = FixedOffset(0, "UTC", -2) met = FixedOffset(60, "MET", 3) t1 = time( 7, 47, tzinfo=est) t2 = time(12, 47, tzinfo=utc) t3 = time(13, 47, tzinfo=met) t4 = time(microsecond=40) t5 = time(microsecond=40, tzinfo=utc) self.assertEqual(t1.tzinfo, est) self.assertEqual(t2.tzinfo, utc) self.assertEqual(t3.tzinfo, met) self.assertIsNone(t4.tzinfo) self.assertEqual(t5.tzinfo, utc) self.assertEqual(t1.utcoffset(), timedelta(minutes=-300)) self.assertEqual(t2.utcoffset(), timedelta(minutes=0)) self.assertEqual(t3.utcoffset(), timedelta(minutes=60)) self.assertIsNone(t4.utcoffset()) self.assertRaises(TypeError, t1.utcoffset, "no args") self.assertEqual(t1.tzname(), "EST") self.assertEqual(t2.tzname(), "UTC") self.assertEqual(t3.tzname(), "MET") self.assertIsNone(t4.tzname()) self.assertRaises(TypeError, t1.tzname, "no args") self.assertEqual(t1.dst(), timedelta(minutes=1)) self.assertEqual(t2.dst(), timedelta(minutes=-2)) self.assertEqual(t3.dst(), timedelta(minutes=3)) self.assertIsNone(t4.dst()) self.assertRaises(TypeError, t1.dst, "no args") self.assertEqual(hash(t1), hash(t2)) self.assertEqual(hash(t1), hash(t3)) self.assertEqual(hash(t2), hash(t3)) self.assertEqual(t1, t2) self.assertEqual(t1, t3) self.assertEqual(t2, t3) self.assertRaises(TypeError, lambda: t4 == t5) # mixed tz-aware & naive self.assertRaises(TypeError, lambda: t4 < t5) # mixed tz-aware & naive self.assertRaises(TypeError, lambda: t5 < t4) # mixed tz-aware & naive self.assertEqual(str(t1), "07:47:00-05:00") self.assertEqual(str(t2), "12:47:00+00:00") self.assertEqual(str(t3), "13:47:00+01:00") self.assertEqual(str(t4), "00:00:00.000040") self.assertEqual(str(t5), "00:00:00.000040+00:00") self.assertEqual(t1.isoformat(), "07:47:00-05:00") self.assertEqual(t2.isoformat(), "12:47:00+00:00") self.assertEqual(t3.isoformat(), "13:47:00+01:00") self.assertEqual(t4.isoformat(), "00:00:00.000040") self.assertEqual(t5.isoformat(), "00:00:00.000040+00:00") d = 'datetime.time' self.assertEqual(repr(t1), d + "(7, 47, tzinfo=est)") self.assertEqual(repr(t2), d + "(12, 47, tzinfo=utc)") self.assertEqual(repr(t3), d + "(13, 47, tzinfo=met)") self.assertEqual(repr(t4), d + "(0, 0, 0, 40)") self.assertEqual(repr(t5), d + "(0, 0, 0, 40, tzinfo=utc)") self.assertEqual(t1.strftime("%H:%M:%S %%Z=%Z %%z=%z"), "07:47:00 %Z=EST %z=-0500") self.assertEqual(t2.strftime("%H:%M:%S %Z %z"), "12:47:00 UTC +0000") self.assertEqual(t3.strftime("%H:%M:%S %Z %z"), "13:47:00 MET +0100") yuck = FixedOffset(-1439, "%z %Z %%z%%Z") t1 = time(23, 59, tzinfo=yuck) self.assertEqual(t1.strftime("%H:%M %%Z='%Z' %%z='%z'"), "23:59 %Z='%z %Z %%z%%Z' %z='-2359'") # Check that an invalid tzname result raises an exception. class Badtzname(tzinfo): def tzname(self, dt): return 42 t = time(2, 3, 4, tzinfo=Badtzname()) self.assertEqual(t.strftime("%H:%M:%S"), "02:03:04") self.assertRaises(TypeError, t.strftime, "%Z") @unittest.expectedFailure def test_hash_edge_cases(self): # Offsets that overflow a basic time. t1 = self.theclass(0, 1, 2, 3, tzinfo=FixedOffset(1439, "")) t2 = self.theclass(0, 0, 2, 3, tzinfo=FixedOffset(1438, "")) self.assertEqual(hash(t1), hash(t2)) t1 = self.theclass(23, 58, 6, 100, tzinfo=FixedOffset(-1000, "")) t2 = self.theclass(23, 48, 6, 100, tzinfo=FixedOffset(-1010, "")) self.assertEqual(hash(t1), hash(t2)) # def test_pickling(self): # # Try one without a tzinfo. # args = 20, 59, 16, 64**2 # orig = self.theclass(*args) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) # # Try one with a tzinfo. # tinfo = PicklableFixedOffset(-300, 'cookie') # orig = self.theclass(5, 6, 7, tzinfo=tinfo) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) # self.assertIsInstance(derived.tzinfo, PicklableFixedOffset) # self.assertEqual(derived.utcoffset(), timedelta(minutes=-300)) # self.assertEqual(derived.tzname(), 'cookie') def test_more_bool(self): # Test cases with non-None tzinfo. cls = self.theclass t = cls(0, tzinfo=FixedOffset(-300, "")) self.assertTrue(t) t = cls(5, tzinfo=FixedOffset(-300, "")) self.assertTrue(t) t = cls(5, tzinfo=FixedOffset(300, "")) self.assertFalse(t) t = cls(23, 59, tzinfo=FixedOffset(23*60 + 59, "")) self.assertFalse(t) # Mostly ensuring this doesn't overflow internally. t = cls(0, tzinfo=FixedOffset(23*60 + 59, "")) self.assertTrue(t) # But this should yield a value error -- the utcoffset is bogus. t = cls(0, tzinfo=FixedOffset(24*60, "")) self.assertRaises(ValueError, lambda: bool(t)) # Likewise. t = cls(0, tzinfo=FixedOffset(-24*60, "")) self.assertRaises(ValueError, lambda: bool(t)) @unittest.expectedFailure def test_replace(self): cls = self.theclass z100 = FixedOffset(100, "+100") zm200 = FixedOffset(timedelta(minutes=-200), "-200") args = [1, 2, 3, 4, z100] base = cls(*args) self.assertEqual(base, base.replace()) i = 0 for name, newval in (("hour", 5), ("minute", 6), ("second", 7), ("microsecond", 8), ("tzinfo", zm200)): newargs = args[:] newargs[i] = newval expected = cls(*newargs) got = base.replace(**{name: newval}) self.assertEqual(expected, got) i += 1 # Ensure we can get rid of a tzinfo. self.assertEqual(base.tzname(), "+100") base2 = base.replace(tzinfo=None) self.assertIsNone(base2.tzinfo) self.assertIsNone(base2.tzname()) # Ensure we can add one. base3 = base2.replace(tzinfo=z100) self.assertEqual(base, base3) self.assertIs(base.tzinfo, base3.tzinfo) # Out of bounds. base = cls(1) self.assertRaises(ValueError, base.replace, hour=24) self.assertRaises(ValueError, base.replace, minute=-1) self.assertRaises(ValueError, base.replace, second=100) self.assertRaises(ValueError, base.replace, microsecond=1000000) @unittest.expectedFailure def test_mixed_compare(self): t1 = time(1, 2, 3) t2 = time(1, 2, 3) self.assertEqual(t1, t2) t2 = t2.replace(tzinfo=None) self.assertEqual(t1, t2) t2 = t2.replace(tzinfo=FixedOffset(None, "")) self.assertEqual(t1, t2) t2 = t2.replace(tzinfo=FixedOffset(0, "")) self.assertRaises(TypeError, lambda: t1 == t2) # In time w/ identical tzinfo objects, utcoffset is ignored. class Varies(tzinfo): def __init__(self): self.offset = timedelta(minutes=22) def utcoffset(self, t): self.offset += timedelta(minutes=1) return self.offset v = Varies() t1 = t2.replace(tzinfo=v) t2 = t2.replace(tzinfo=v) self.assertEqual(t1.utcoffset(), timedelta(minutes=23)) self.assertEqual(t2.utcoffset(), timedelta(minutes=24)) self.assertEqual(t1, t2) # But if they're not identical, it isn't ignored. t2 = t2.replace(tzinfo=Varies()) self.assertTrue(t1 < t2) # t1's offset counter still going up @unittest.expectedFailure def test_subclass_timetz(self): class C(self.theclass): theAnswer = 42 def __new__(cls, *args, **kws): temp = kws.copy() extra = temp.pop('extra') result = self.theclass.__new__(cls, *args, **temp) result.extra = extra return result def newmeth(self, start): return start + self.hour + self.second args = 4, 5, 6, 500, FixedOffset(-300, "EST", 1) dt1 = self.theclass(*args) dt2 = C(*args, **{'extra': 7}) self.assertEqual(dt2.__class__, C) self.assertEqual(dt2.theAnswer, 42) self.assertEqual(dt2.extra, 7) self.assertEqual(dt1.utcoffset(), dt2.utcoffset()) self.assertEqual(dt2.newmeth(-7), dt1.hour + dt1.second - 7) # Testing datetime objects with a non-None tzinfo. class TestDateTimeTZ(TestDateTime, TZInfoBase, unittest.TestCase): theclass = datetime def test_trivial(self): dt = self.theclass(1, 2, 3, 4, 5, 6, 7) self.assertEqual(dt.year, 1) self.assertEqual(dt.month, 2) self.assertEqual(dt.day, 3) self.assertEqual(dt.hour, 4) self.assertEqual(dt.minute, 5) self.assertEqual(dt.second, 6) self.assertEqual(dt.microsecond, 7) self.assertEqual(dt.tzinfo, None) def test_even_more_compare(self): # The test_compare() and test_more_compare() inherited from TestDate # and TestDateTime covered non-tzinfo cases. # Smallest possible after UTC adjustment. t1 = self.theclass(1, 1, 1, tzinfo=FixedOffset(1439, "")) # Largest possible after UTC adjustment. t2 = self.theclass(MAXYEAR, 12, 31, 23, 59, 59, 999999, tzinfo=FixedOffset(-1439, "")) # Make sure those compare correctly, and w/o overflow. self.assertTrue(t1 < t2) self.assertTrue(t1 != t2) self.assertTrue(t2 > t1) self.assertTrue(t1 == t1) self.assertTrue(t2 == t2) # Equal afer adjustment. t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(1, "")) t2 = self.theclass(2, 1, 1, 3, 13, tzinfo=FixedOffset(3*60+13+2, "")) self.assertEqual(t1, t2) # Change t1 not to subtract a minute, and t1 should be larger. t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(0, "")) self.assertTrue(t1 > t2) # Change t1 to subtract 2 minutes, and t1 should be smaller. t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(2, "")) self.assertTrue(t1 < t2) # Back to the original t1, but make seconds resolve it. t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(1, ""), second=1) self.assertTrue(t1 > t2) # Likewise, but make microseconds resolve it. t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(1, ""), microsecond=1) self.assertTrue(t1 > t2) # Make t2 naive and it should fail. t2 = self.theclass.min self.assertRaises(TypeError, lambda: t1 == t2) self.assertEqual(t2, t2) # It's also naive if it has tzinfo but tzinfo.utcoffset() is None. class Naive(tzinfo): def utcoffset(self, dt): return None t2 = self.theclass(5, 6, 7, tzinfo=Naive()) self.assertRaises(TypeError, lambda: t1 == t2) self.assertEqual(t2, t2) # OTOH, it's OK to compare two of these mixing the two ways of being # naive. t1 = self.theclass(5, 6, 7) self.assertEqual(t1, t2) # Try a bogus uctoffset. class Bogus(tzinfo): def utcoffset(self, dt): return timedelta(minutes=1440) # out of bounds t1 = self.theclass(2, 2, 2, tzinfo=Bogus()) t2 = self.theclass(2, 2, 2, tzinfo=FixedOffset(0, "")) self.assertRaises(ValueError, lambda: t1 == t2) # def test_pickling(self): # # Try one without a tzinfo. # args = 6, 7, 23, 20, 59, 1, 64**2 # orig = self.theclass(*args) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) # # Try one with a tzinfo. # tinfo = PicklableFixedOffset(-300, 'cookie') # orig = self.theclass(*args, **{'tzinfo': tinfo}) # derived = self.theclass(1, 1, 1, tzinfo=FixedOffset(0, "", 0)) # for pickler, unpickler, proto in pickle_choices: # green = pickler.dumps(orig, proto) # derived = unpickler.loads(green) # self.assertEqual(orig, derived) # self.assertIsInstance(derived.tzinfo, PicklableFixedOffset) # self.assertEqual(derived.utcoffset(), timedelta(minutes=-300)) # self.assertEqual(derived.tzname(), 'cookie') def test_extreme_hashes(self): # If an attempt is made to hash these via subtracting the offset # then hashing a datetime object, OverflowError results. The # Python implementation used to blow up here. t = self.theclass(1, 1, 1, tzinfo=FixedOffset(1439, "")) hash(t) t = self.theclass(MAXYEAR, 12, 31, 23, 59, 59, 999999, tzinfo=FixedOffset(-1439, "")) hash(t) # OTOH, an OOB offset should blow up. t = self.theclass(5, 5, 5, tzinfo=FixedOffset(-1440, "")) self.assertRaises(ValueError, hash, t) @unittest.expectedFailure def test_zones(self): est = FixedOffset(-300, "EST") utc = FixedOffset(0, "UTC") met = FixedOffset(60, "MET") t1 = datetime(2002, 3, 19, 7, 47, tzinfo=est) t2 = datetime(2002, 3, 19, 12, 47, tzinfo=utc) t3 = datetime(2002, 3, 19, 13, 47, tzinfo=met) self.assertEqual(t1.tzinfo, est) self.assertEqual(t2.tzinfo, utc) self.assertEqual(t3.tzinfo, met) self.assertEqual(t1.utcoffset(), timedelta(minutes=-300)) self.assertEqual(t2.utcoffset(), timedelta(minutes=0)) self.assertEqual(t3.utcoffset(), timedelta(minutes=60)) self.assertEqual(t1.tzname(), "EST") self.assertEqual(t2.tzname(), "UTC") self.assertEqual(t3.tzname(), "MET") self.assertEqual(hash(t1), hash(t2)) self.assertEqual(hash(t1), hash(t3)) self.assertEqual(hash(t2), hash(t3)) self.assertEqual(t1, t2) self.assertEqual(t1, t3) self.assertEqual(t2, t3) self.assertEqual(str(t1), "2002-03-19 07:47:00-05:00") self.assertEqual(str(t2), "2002-03-19 12:47:00+00:00") self.assertEqual(str(t3), "2002-03-19 13:47:00+01:00") d = 'datetime.datetime(2002, 3, 19, ' self.assertEqual(repr(t1), d + "7, 47, tzinfo=est)") self.assertEqual(repr(t2), d + "12, 47, tzinfo=utc)") self.assertEqual(repr(t3), d + "13, 47, tzinfo=met)") def test_combine(self): met = FixedOffset(60, "MET") d = date(2002, 3, 4) tz = time(18, 45, 3, 1234, tzinfo=met) dt = datetime.combine(d, tz) self.assertEqual(dt, datetime(2002, 3, 4, 18, 45, 3, 1234, tzinfo=met)) def test_extract(self): met = FixedOffset(60, "MET") dt = self.theclass(2002, 3, 4, 18, 45, 3, 1234, tzinfo=met) self.assertEqual(dt.date(), date(2002, 3, 4)) self.assertEqual(dt.time(), time(18, 45, 3, 1234)) self.assertEqual(dt.timetz(), time(18, 45, 3, 1234, tzinfo=met)) @unittest.expectedFailure def test_tz_aware_arithmetic(self): import random now = self.theclass.now() tz55 = FixedOffset(-330, "west 5:30") timeaware = now.time().replace(tzinfo=tz55) nowaware = self.theclass.combine(now.date(), timeaware) self.assertIs(nowaware.tzinfo, tz55) self.assertEqual(nowaware.timetz(), timeaware) # Can't mix aware and non-aware. self.assertRaises(TypeError, lambda: now - nowaware) self.assertRaises(TypeError, lambda: nowaware - now) # And adding datetime's doesn't make sense, aware or not. self.assertRaises(TypeError, lambda: now + nowaware) self.assertRaises(TypeError, lambda: nowaware + now) self.assertRaises(TypeError, lambda: nowaware + nowaware) # Subtracting should yield 0. self.assertEqual(now - now, timedelta(0)) self.assertEqual(nowaware - nowaware, timedelta(0)) # Adding a delta should preserve tzinfo. delta = timedelta(weeks=1, minutes=12, microseconds=5678) nowawareplus = nowaware + delta self.assertIs(nowaware.tzinfo, tz55) nowawareplus2 = delta + nowaware self.assertIs(nowawareplus2.tzinfo, tz55) self.assertEqual(nowawareplus, nowawareplus2) # that - delta should be what we started with, and that - what we # started with should be delta. diff = nowawareplus - delta self.assertIs(diff.tzinfo, tz55) self.assertEqual(nowaware, diff) self.assertRaises(TypeError, lambda: delta - nowawareplus) self.assertEqual(nowawareplus - nowaware, delta) # Make up a random timezone. tzr = FixedOffset(random.randrange(-1439, 1440), "randomtimezone") # Attach it to nowawareplus. nowawareplus = nowawareplus.replace(tzinfo=tzr) self.assertIs(nowawareplus.tzinfo, tzr) # Make sure the difference takes the timezone adjustments into account. got = nowaware - nowawareplus # Expected: (nowaware base - nowaware offset) - # (nowawareplus base - nowawareplus offset) = # (nowaware base - nowawareplus base) + # (nowawareplus offset - nowaware offset) = # -delta + nowawareplus offset - nowaware offset expected = nowawareplus.utcoffset() - nowaware.utcoffset() - delta self.assertEqual(got, expected) # Try max possible difference. min = self.theclass(1, 1, 1, tzinfo=FixedOffset(1439, "min")) max = self.theclass(MAXYEAR, 12, 31, 23, 59, 59, 999999, tzinfo=FixedOffset(-1439, "max")) maxdiff = max - min self.assertEqual(maxdiff, self.theclass.max - self.theclass.min + timedelta(minutes=2*1439)) @unittest.expectedFailure def test_tzinfo_now(self): meth = self.theclass.now # Ensure it doesn't require tzinfo (i.e., that this doesn't blow up). base = meth() # Try with and without naming the keyword. off42 = FixedOffset(42, "42") another = meth(off42) again = meth(tz=off42) self.assertIs(another.tzinfo, again.tzinfo) self.assertEqual(another.utcoffset(), timedelta(minutes=42)) # Bad argument with and w/o naming the keyword. self.assertRaises(TypeError, meth, 16) self.assertRaises(TypeError, meth, tzinfo=16) # Bad keyword name. self.assertRaises(TypeError, meth, tinfo=off42) # Too many args. self.assertRaises(TypeError, meth, off42, off42) # We don't know which time zone we're in, and don't have a tzinfo # class to represent it, so seeing whether a tz argument actually # does a conversion is tricky. weirdtz = FixedOffset(timedelta(hours=15, minutes=58), "weirdtz", 0) utc = FixedOffset(0, "utc", 0) for dummy in range(3): now = datetime.now(weirdtz) self.assertIs(now.tzinfo, weirdtz) utcnow = datetime.utcnow().replace(tzinfo=utc) now2 = utcnow.astimezone(weirdtz) if abs(now - now2) < timedelta(seconds=30): break # Else the code is broken, or more than 30 seconds passed between # calls; assuming the latter, just try again. else: # Three strikes and we're out. self.fail("utcnow(), now(tz), or astimezone() may be broken") @unittest.expectedFailure def test_tzinfo_fromtimestamp(self): import time meth = self.theclass.fromtimestamp ts = time.time() # Ensure it doesn't require tzinfo (i.e., that this doesn't blow up). base = meth(ts) # Try with and without naming the keyword. off42 = FixedOffset(42, "42") another = meth(ts, off42) again = meth(ts, tz=off42) self.assertIs(another.tzinfo, again.tzinfo) self.assertEqual(another.utcoffset(), timedelta(minutes=42)) # Bad argument with and w/o naming the keyword. self.assertRaises(TypeError, meth, ts, 16) self.assertRaises(TypeError, meth, ts, tzinfo=16) # Bad keyword name. self.assertRaises(TypeError, meth, ts, tinfo=off42) # Too many args. self.assertRaises(TypeError, meth, ts, off42, off42) # Too few args. self.assertRaises(TypeError, meth) # Try to make sure tz= actually does some conversion. timestamp = 1000000000 utcdatetime = datetime.utcfromtimestamp(timestamp) # In POSIX (epoch 1970), that's 2001-09-09 01:46:40 UTC, give or take. # But on some flavor of Mac, it's nowhere near that. So we can't have # any idea here what time that actually is, we can only test that # relative changes match. utcoffset = timedelta(hours=-15, minutes=39) # arbitrary, but not zero tz = FixedOffset(utcoffset, "tz", 0) expected = utcdatetime + utcoffset got = datetime.fromtimestamp(timestamp, tz) self.assertEqual(expected, got.replace(tzinfo=None)) def test_tzinfo_utcnow(self): meth = self.theclass.utcnow # Ensure it doesn't require tzinfo (i.e., that this doesn't blow up). base = meth() # Try with and without naming the keyword; for whatever reason, # utcnow() doesn't accept a tzinfo argument. off42 = FixedOffset(42, "42") self.assertRaises(TypeError, meth, off42) self.assertRaises(TypeError, meth, tzinfo=off42) def test_tzinfo_utcfromtimestamp(self): import time meth = self.theclass.utcfromtimestamp ts = time.time() # Ensure it doesn't require tzinfo (i.e., that this doesn't blow up). base = meth(ts) # Try with and without naming the keyword; for whatever reason, # utcfromtimestamp() doesn't accept a tzinfo argument. off42 = FixedOffset(42, "42") self.assertRaises(TypeError, meth, ts, off42) self.assertRaises(TypeError, meth, ts, tzinfo=off42) def test_tzinfo_timetuple(self): # TestDateTime tested most of this. datetime adds a twist to the # DST flag. class DST(tzinfo): def __init__(self, dstvalue): if isinstance(dstvalue, int): dstvalue = timedelta(minutes=dstvalue) self.dstvalue = dstvalue def dst(self, dt): return self.dstvalue cls = self.theclass for dstvalue, flag in (-33, 1), (33, 1), (0, 0), (None, -1): d = cls(1, 1, 1, 10, 20, 30, 40, tzinfo=DST(dstvalue)) t = d.timetuple() self.assertEqual(1, t.tm_year) self.assertEqual(1, t.tm_mon) self.assertEqual(1, t.tm_mday) self.assertEqual(10, t.tm_hour) self.assertEqual(20, t.tm_min) self.assertEqual(30, t.tm_sec) self.assertEqual(0, t.tm_wday) self.assertEqual(1, t.tm_yday) self.assertEqual(flag, t.tm_isdst) # dst() returns wrong type. self.assertRaises(TypeError, cls(1, 1, 1, tzinfo=DST("x")).timetuple) # dst() at the edge. self.assertEqual(cls(1,1,1, tzinfo=DST(1439)).timetuple().tm_isdst, 1) self.assertEqual(cls(1,1,1, tzinfo=DST(-1439)).timetuple().tm_isdst, 1) # dst() out of range. self.assertRaises(ValueError, cls(1,1,1, tzinfo=DST(1440)).timetuple) self.assertRaises(ValueError, cls(1,1,1, tzinfo=DST(-1440)).timetuple) def test_utctimetuple(self): class DST(tzinfo): def __init__(self, dstvalue): if isinstance(dstvalue, int): dstvalue = timedelta(minutes=dstvalue) self.dstvalue = dstvalue def dst(self, dt): return self.dstvalue cls = self.theclass # This can't work: DST didn't implement utcoffset. self.assertRaises(NotImplementedError, cls(1, 1, 1, tzinfo=DST(0)).utcoffset) class UOFS(DST): def __init__(self, uofs, dofs=None): DST.__init__(self, dofs) self.uofs = timedelta(minutes=uofs) def utcoffset(self, dt): return self.uofs # Ensure tm_isdst is 0 regardless of what dst() says: DST is never # in effect for a UTC time. for dstvalue in -33, 33, 0, None: d = cls(1, 2, 3, 10, 20, 30, 40, tzinfo=UOFS(-53, dstvalue)) t = d.utctimetuple() self.assertEqual(d.year, t.tm_year) self.assertEqual(d.month, t.tm_mon) self.assertEqual(d.day, t.tm_mday) self.assertEqual(11, t.tm_hour) # 20mm + 53mm = 1hn + 13mm self.assertEqual(13, t.tm_min) self.assertEqual(d.second, t.tm_sec) self.assertEqual(d.weekday(), t.tm_wday) self.assertEqual(d.toordinal() - date(1, 1, 1).toordinal() + 1, t.tm_yday) self.assertEqual(0, t.tm_isdst) # At the edges, UTC adjustment can normalize into years out-of-range # for a datetime object. Ensure that a correct timetuple is # created anyway. tiny = cls(MINYEAR, 1, 1, 0, 0, 37, tzinfo=UOFS(1439)) # That goes back 1 minute less than a full day. t = tiny.utctimetuple() self.assertEqual(t.tm_year, MINYEAR-1) self.assertEqual(t.tm_mon, 12) self.assertEqual(t.tm_mday, 31) self.assertEqual(t.tm_hour, 0) self.assertEqual(t.tm_min, 1) self.assertEqual(t.tm_sec, 37) self.assertEqual(t.tm_yday, 366) # "year 0" is a leap year self.assertEqual(t.tm_isdst, 0) huge = cls(MAXYEAR, 12, 31, 23, 59, 37, 999999, tzinfo=UOFS(-1439)) # That goes forward 1 minute less than a full day. t = huge.utctimetuple() self.assertEqual(t.tm_year, MAXYEAR+1) self.assertEqual(t.tm_mon, 1) self.assertEqual(t.tm_mday, 1) self.assertEqual(t.tm_hour, 23) self.assertEqual(t.tm_min, 58) self.assertEqual(t.tm_sec, 37) self.assertEqual(t.tm_yday, 1) self.assertEqual(t.tm_isdst, 0) @unittest.expectedFailure def test_tzinfo_isoformat(self): zero = FixedOffset(0, "+00:00") plus = FixedOffset(220, "+03:40") minus = FixedOffset(-231, "-03:51") unknown = FixedOffset(None, "") cls = self.theclass datestr = '0001-02-03' for ofs in None, zero, plus, minus, unknown: for us in 0, 987001: d = cls(1, 2, 3, 4, 5, 59, us, tzinfo=ofs) timestr = '04:05:59' + (us and '.987001' or '') ofsstr = ofs is not None and d.tzname() or '' tailstr = timestr + ofsstr iso = d.isoformat() self.assertEqual(iso, datestr + 'T' + tailstr) self.assertEqual(iso, d.isoformat('T')) self.assertEqual(d.isoformat('k'), datestr + 'k' + tailstr) self.assertEqual(str(d), datestr + ' ' + tailstr) @unittest.expectedFailure def test_replace(self): cls = self.theclass z100 = FixedOffset(100, "+100") zm200 = FixedOffset(timedelta(minutes=-200), "-200") args = [1, 2, 3, 4, 5, 6, 7, z100] base = cls(*args) self.assertEqual(base, base.replace()) i = 0 for name, newval in (("year", 2), ("month", 3), ("day", 4), ("hour", 5), ("minute", 6), ("second", 7), ("microsecond", 8), ("tzinfo", zm200)): newargs = args[:] newargs[i] = newval expected = cls(*newargs) got = base.replace(**{name: newval}) self.assertEqual(expected, got) i += 1 # Ensure we can get rid of a tzinfo. self.assertEqual(base.tzname(), "+100") base2 = base.replace(tzinfo=None) self.assertIsNone(base2.tzinfo) self.assertIsNone(base2.tzname()) # Ensure we can add one. base3 = base2.replace(tzinfo=z100) self.assertEqual(base, base3) self.assertIs(base.tzinfo, base3.tzinfo) # Out of bounds. base = cls(2000, 2, 29) self.assertRaises(ValueError, base.replace, year=2001) @unittest.expectedFailure def test_more_astimezone(self): # The inherited test_astimezone covered some trivial and error cases. fnone = FixedOffset(None, "None") f44m = FixedOffset(44, "44") fm5h = FixedOffset(-timedelta(hours=5), "m300") dt = self.theclass.now(tz=f44m) self.assertIs(dt.tzinfo, f44m) # Replacing with degenerate tzinfo raises an exception. self.assertRaises(ValueError, dt.astimezone, fnone) # Ditto with None tz. self.assertRaises(TypeError, dt.astimezone, None) # Replacing with same tzinfo makes no change. x = dt.astimezone(dt.tzinfo) self.assertIs(x.tzinfo, f44m) self.assertEqual(x.date(), dt.date()) self.assertEqual(x.time(), dt.time()) # Replacing with different tzinfo does adjust. got = dt.astimezone(fm5h) self.assertIs(got.tzinfo, fm5h) self.assertEqual(got.utcoffset(), timedelta(hours=-5)) expected = dt - dt.utcoffset() # in effect, convert to UTC expected += fm5h.utcoffset(dt) # and from there to local time expected = expected.replace(tzinfo=fm5h) # and attach new tzinfo self.assertEqual(got.date(), expected.date()) self.assertEqual(got.time(), expected.time()) self.assertEqual(got.timetz(), expected.timetz()) self.assertIs(got.tzinfo, expected.tzinfo) self.assertEqual(got, expected) @unittest.expectedFailure def test_aware_subtract(self): cls = self.theclass # Ensure that utcoffset() is ignored when the operands have the # same tzinfo member. class OperandDependentOffset(tzinfo): def utcoffset(self, t): if t.minute < 10: # d0 and d1 equal after adjustment return timedelta(minutes=t.minute) else: # d2 off in the weeds return timedelta(minutes=59) base = cls(8, 9, 10, 11, 12, 13, 14, tzinfo=OperandDependentOffset()) d0 = base.replace(minute=3) d1 = base.replace(minute=9) d2 = base.replace(minute=11) for x in d0, d1, d2: for y in d0, d1, d2: got = x - y expected = timedelta(minutes=x.minute - y.minute) self.assertEqual(got, expected) # OTOH, if the tzinfo members are distinct, utcoffsets aren't # ignored. base = cls(8, 9, 10, 11, 12, 13, 14) d0 = base.replace(minute=3, tzinfo=OperandDependentOffset()) d1 = base.replace(minute=9, tzinfo=OperandDependentOffset()) d2 = base.replace(minute=11, tzinfo=OperandDependentOffset()) for x in d0, d1, d2: for y in d0, d1, d2: got = x - y if (x is d0 or x is d1) and (y is d0 or y is d1): expected = timedelta(0) elif x is y is d2: expected = timedelta(0) elif x is d2: expected = timedelta(minutes=(11-59)-0) else: assert y is d2 expected = timedelta(minutes=0-(11-59)) self.assertEqual(got, expected) @unittest.expectedFailure def test_mixed_compare(self): t1 = datetime(1, 2, 3, 4, 5, 6, 7) t2 = datetime(1, 2, 3, 4, 5, 6, 7) self.assertEqual(t1, t2) t2 = t2.replace(tzinfo=None) self.assertEqual(t1, t2) t2 = t2.replace(tzinfo=FixedOffset(None, "")) self.assertEqual(t1, t2) t2 = t2.replace(tzinfo=FixedOffset(0, "")) self.assertRaises(TypeError, lambda: t1 == t2) # In datetime w/ identical tzinfo objects, utcoffset is ignored. class Varies(tzinfo): def __init__(self): self.offset = timedelta(minutes=22) def utcoffset(self, t): self.offset += timedelta(minutes=1) return self.offset v = Varies() t1 = t2.replace(tzinfo=v) t2 = t2.replace(tzinfo=v) self.assertEqual(t1.utcoffset(), timedelta(minutes=23)) self.assertEqual(t2.utcoffset(), timedelta(minutes=24)) self.assertEqual(t1, t2) # But if they're not identical, it isn't ignored. t2 = t2.replace(tzinfo=Varies()) self.assertTrue(t1 < t2) # t1's offset counter still going up @unittest.expectedFailure def test_subclass_datetimetz(self): class C(self.theclass): theAnswer = 42 def __new__(cls, *args, **kws): temp = kws.copy() extra = temp.pop('extra') result = self.theclass.__new__(cls, *args, **temp) result.extra = extra return result def newmeth(self, start): return start + self.hour + self.year args = 2002, 12, 31, 4, 5, 6, 500, FixedOffset(-300, "EST", 1) dt1 = self.theclass(*args) dt2 = C(*args, **{'extra': 7}) self.assertEqual(dt2.__class__, C) self.assertEqual(dt2.theAnswer, 42) self.assertEqual(dt2.extra, 7) self.assertEqual(dt1.utcoffset(), dt2.utcoffset()) self.assertEqual(dt2.newmeth(-7), dt1.hour + dt1.year - 7) # Pain to set up DST-aware tzinfo classes. def first_sunday_on_or_after(dt): days_to_go = 6 - dt.weekday() if days_to_go: dt += timedelta(days_to_go) return dt ZERO = timedelta(0) HOUR = timedelta(hours=1) DAY = timedelta(days=1) # In the US, DST starts at 2am (standard time) on the first Sunday in April. DSTSTART = datetime(1, 4, 1, 2) # and ends at 2am (DST time; 1am standard time) on the last Sunday of Oct, # which is the first Sunday on or after Oct 25. Because we view 1:MM as # being standard time on that day, there is no spelling in local time of # the last hour of DST (that's 1:MM DST, but 1:MM is taken as standard time). DSTEND = datetime(1, 10, 25, 1) class USTimeZone(tzinfo): def __init__(self, hours, reprname, stdname, dstname): self.stdoffset = timedelta(hours=hours) self.reprname = reprname self.stdname = stdname self.dstname = dstname def __repr__(self): return self.reprname def tzname(self, dt): if self.dst(dt): return self.dstname else: return self.stdname def utcoffset(self, dt): return self.stdoffset + self.dst(dt) def dst(self, dt): if dt is None or dt.tzinfo is None: # An exception instead may be sensible here, in one or more of # the cases. return ZERO assert dt.tzinfo is self # Find first Sunday in April. start = first_sunday_on_or_after(DSTSTART.replace(year=dt.year)) assert start.weekday() == 6 and start.month == 4 and start.day <= 7 # Find last Sunday in October. end = first_sunday_on_or_after(DSTEND.replace(year=dt.year)) assert end.weekday() == 6 and end.month == 10 and end.day >= 25 # Can't compare naive to aware objects, so strip the timezone from # dt first. if start <= dt.replace(tzinfo=None) < end: return HOUR else: return ZERO Eastern = USTimeZone(-5, "Eastern", "EST", "EDT") Central = USTimeZone(-6, "Central", "CST", "CDT") Mountain = USTimeZone(-7, "Mountain", "MST", "MDT") Pacific = USTimeZone(-8, "Pacific", "PST", "PDT") utc_real = FixedOffset(0, "UTC", 0) # For better test coverage, we want another flavor of UTC that's west of # the Eastern and Pacific timezones. utc_fake = FixedOffset(-12*60, "UTCfake", 0) class TestTimezoneConversions(unittest.TestCase): # The DST switch times for 2002, in std time. dston = datetime(2002, 4, 7, 2) dstoff = datetime(2002, 10, 27, 1) theclass = datetime # Check a time that's inside DST. def checkinside(self, dt, tz, utc, dston, dstoff): self.assertEqual(dt.dst(), HOUR) # Conversion to our own timezone is always an identity. self.assertEqual(dt.astimezone(tz), dt) asutc = dt.astimezone(utc) there_and_back = asutc.astimezone(tz) # Conversion to UTC and back isn't always an identity here, # because there are redundant spellings (in local time) of # UTC time when DST begins: the clock jumps from 1:59:59 # to 3:00:00, and a local time of 2:MM:SS doesn't really # make sense then. The classes above treat 2:MM:SS as # daylight time then (it's "after 2am"), really an alias # for 1:MM:SS standard time. The latter form is what # conversion back from UTC produces. if dt.date() == dston.date() and dt.hour == 2: # We're in the redundant hour, and coming back from # UTC gives the 1:MM:SS standard-time spelling. self.assertEqual(there_and_back + HOUR, dt) # Although during was considered to be in daylight # time, there_and_back is not. self.assertEqual(there_and_back.dst(), ZERO) # They're the same times in UTC. self.assertEqual(there_and_back.astimezone(utc), dt.astimezone(utc)) else: # We're not in the redundant hour. self.assertEqual(dt, there_and_back) # Because we have a redundant spelling when DST begins, there is # (unfortunately) an hour when DST ends that can't be spelled at all in # local time. When DST ends, the clock jumps from 1:59 back to 1:00 # again. The hour 1:MM DST has no spelling then: 1:MM is taken to be # standard time. 1:MM DST == 0:MM EST, but 0:MM is taken to be # daylight time. The hour 1:MM daylight == 0:MM standard can't be # expressed in local time. Nevertheless, we want conversion back # from UTC to mimic the local clock's "repeat an hour" behavior. nexthour_utc = asutc + HOUR nexthour_tz = nexthour_utc.astimezone(tz) if dt.date() == dstoff.date() and dt.hour == 0: # We're in the hour before the last DST hour. The last DST hour # is ineffable. We want the conversion back to repeat 1:MM. self.assertEqual(nexthour_tz, dt.replace(hour=1)) nexthour_utc += HOUR nexthour_tz = nexthour_utc.astimezone(tz) self.assertEqual(nexthour_tz, dt.replace(hour=1)) else: self.assertEqual(nexthour_tz - dt, HOUR) # Check a time that's outside DST. def checkoutside(self, dt, tz, utc): self.assertEqual(dt.dst(), ZERO) # Conversion to our own timezone is always an identity. self.assertEqual(dt.astimezone(tz), dt) # Converting to UTC and back is an identity too. asutc = dt.astimezone(utc) there_and_back = asutc.astimezone(tz) self.assertEqual(dt, there_and_back) def convert_between_tz_and_utc(self, tz, utc): dston = self.dston.replace(tzinfo=tz) # Because 1:MM on the day DST ends is taken as being standard time, # there is no spelling in tz for the last hour of daylight time. # For purposes of the test, the last hour of DST is 0:MM, which is # taken as being daylight time (and 1:MM is taken as being standard # time). dstoff = self.dstoff.replace(tzinfo=tz) for delta in (timedelta(weeks=13), DAY, HOUR, timedelta(minutes=1), timedelta(microseconds=1)): self.checkinside(dston, tz, utc, dston, dstoff) for during in dston + delta, dstoff - delta: self.checkinside(during, tz, utc, dston, dstoff) self.checkoutside(dstoff, tz, utc) for outside in dston - delta, dstoff + delta: self.checkoutside(outside, tz, utc) @unittest.expectedFailure def test_easy(self): # Despite the name of this test, the endcases are excruciating. self.convert_between_tz_and_utc(Eastern, utc_real) self.convert_between_tz_and_utc(Pacific, utc_real) self.convert_between_tz_and_utc(Eastern, utc_fake) self.convert_between_tz_and_utc(Pacific, utc_fake) # The next is really dancing near the edge. It works because # Pacific and Eastern are far enough apart that their "problem # hours" don't overlap. self.convert_between_tz_and_utc(Eastern, Pacific) self.convert_between_tz_and_utc(Pacific, Eastern) # OTOH, these fail! Don't enable them. The difficulty is that # the edge case tests assume that every hour is representable in # the "utc" class. This is always true for a fixed-offset tzinfo # class (lke utc_real and utc_fake), but not for Eastern or Central. # For these adjacent DST-aware time zones, the range of time offsets # tested ends up creating hours in the one that aren't representable # in the other. For the same reason, we would see failures in the # Eastern vs Pacific tests too if we added 3*HOUR to the list of # offset deltas in convert_between_tz_and_utc(). # # self.convert_between_tz_and_utc(Eastern, Central) # can't work # self.convert_between_tz_and_utc(Central, Eastern) # can't work @unittest.expectedFailure def test_tricky(self): # 22:00 on day before daylight starts. fourback = self.dston - timedelta(hours=4) ninewest = FixedOffset(-9*60, "-0900", 0) fourback = fourback.replace(tzinfo=ninewest) # 22:00-0900 is 7:00 UTC == 2:00 EST == 3:00 DST. Since it's "after # 2", we should get the 3 spelling. # If we plug 22:00 the day before into Eastern, it "looks like std # time", so its offset is returned as -5, and -5 - -9 = 4. Adding 4 # to 22:00 lands on 2:00, which makes no sense in local time (the # local clock jumps from 1 to 3). The point here is to make sure we # get the 3 spelling. expected = self.dston.replace(hour=3) got = fourback.astimezone(Eastern).replace(tzinfo=None) self.assertEqual(expected, got) # Similar, but map to 6:00 UTC == 1:00 EST == 2:00 DST. In that # case we want the 1:00 spelling. sixutc = self.dston.replace(hour=6, tzinfo=utc_real) # Now 6:00 "looks like daylight", so the offset wrt Eastern is -4, # and adding -4-0 == -4 gives the 2:00 spelling. We want the 1:00 EST # spelling. expected = self.dston.replace(hour=1) got = sixutc.astimezone(Eastern).replace(tzinfo=None) self.assertEqual(expected, got) # Now on the day DST ends, we want "repeat an hour" behavior. # UTC 4:MM 5:MM 6:MM 7:MM checking these # EST 23:MM 0:MM 1:MM 2:MM # EDT 0:MM 1:MM 2:MM 3:MM # wall 0:MM 1:MM 1:MM 2:MM against these for utc in utc_real, utc_fake: for tz in Eastern, Pacific: first_std_hour = self.dstoff - timedelta(hours=2) # 23:MM # Convert that to UTC. first_std_hour -= tz.utcoffset(None) # Adjust for possibly fake UTC. asutc = first_std_hour + utc.utcoffset(None) # First UTC hour to convert; this is 4:00 when utc=utc_real & # tz=Eastern. asutcbase = asutc.replace(tzinfo=utc) for tzhour in (0, 1, 1, 2): expectedbase = self.dstoff.replace(hour=tzhour) for minute in 0, 30, 59: expected = expectedbase.replace(minute=minute) asutc = asutcbase.replace(minute=minute) astz = asutc.astimezone(tz) self.assertEqual(astz.replace(tzinfo=None), expected) asutcbase += HOUR @unittest.expectedFailure def test_bogus_dst(self): class ok(tzinfo): def utcoffset(self, dt): return HOUR def dst(self, dt): return HOUR now = self.theclass.now().replace(tzinfo=utc_real) # Doesn't blow up. now.astimezone(ok()) # Does blow up. class notok(ok): def dst(self, dt): return None self.assertRaises(ValueError, now.astimezone, notok()) @unittest.expectedFailure def test_fromutc(self): self.assertRaises(TypeError, Eastern.fromutc) # not enough args now = datetime.utcnow().replace(tzinfo=utc_real) self.assertRaises(ValueError, Eastern.fromutc, now) # wrong tzinfo now = now.replace(tzinfo=Eastern) # insert correct tzinfo enow = Eastern.fromutc(now) # doesn't blow up self.assertEqual(enow.tzinfo, Eastern) # has right tzinfo member self.assertRaises(TypeError, Eastern.fromutc, now, now) # too many args self.assertRaises(TypeError, Eastern.fromutc, date.today()) # wrong type # Always converts UTC to standard time. class FauxUSTimeZone(USTimeZone): def fromutc(self, dt): return dt + self.stdoffset FEastern = FauxUSTimeZone(-5, "FEastern", "FEST", "FEDT") # UTC 4:MM 5:MM 6:MM 7:MM 8:MM 9:MM # EST 23:MM 0:MM 1:MM 2:MM 3:MM 4:MM # EDT 0:MM 1:MM 2:MM 3:MM 4:MM 5:MM # Check around DST start. start = self.dston.replace(hour=4, tzinfo=Eastern) fstart = start.replace(tzinfo=FEastern) for wall in 23, 0, 1, 3, 4, 5: expected = start.replace(hour=wall) if wall == 23: expected -= timedelta(days=1) got = Eastern.fromutc(start) self.assertEqual(expected, got) expected = fstart + FEastern.stdoffset got = FEastern.fromutc(fstart) self.assertEqual(expected, got) # Ensure astimezone() calls fromutc() too. got = fstart.replace(tzinfo=utc_real).astimezone(FEastern) self.assertEqual(expected, got) start += HOUR fstart += HOUR # Check around DST end. start = self.dstoff.replace(hour=4, tzinfo=Eastern) fstart = start.replace(tzinfo=FEastern) for wall in 0, 1, 1, 2, 3, 4: expected = start.replace(hour=wall) got = Eastern.fromutc(start) self.assertEqual(expected, got) expected = fstart + FEastern.stdoffset got = FEastern.fromutc(fstart) self.assertEqual(expected, got) # Ensure astimezone() calls fromutc() too. got = fstart.replace(tzinfo=utc_real).astimezone(FEastern) self.assertEqual(expected, got) start += HOUR fstart += HOUR ############################################################################# # oddballs class Oddballs(unittest.TestCase): @unittest.expectedFailure def test_bug_1028306(self): # Trying to compare a date to a datetime should act like a mixed- # type comparison, despite that datetime is a subclass of date. as_date = date.today() as_datetime = datetime.combine(as_date, time()) self.assertTrue(as_date != as_datetime) self.assertTrue(as_datetime != as_date) self.assertFalse(as_date == as_datetime) self.assertFalse(as_datetime == as_date) self.assertRaises(TypeError, lambda: as_date < as_datetime) self.assertRaises(TypeError, lambda: as_datetime < as_date) self.assertRaises(TypeError, lambda: as_date <= as_datetime) self.assertRaises(TypeError, lambda: as_datetime <= as_date) self.assertRaises(TypeError, lambda: as_date > as_datetime) self.assertRaises(TypeError, lambda: as_datetime > as_date) self.assertRaises(TypeError, lambda: as_date >= as_datetime) self.assertRaises(TypeError, lambda: as_datetime >= as_date) # Neverthelss, comparison should work with the base-class (date) # projection if use of a date method is forced. self.assertTrue(as_date.__eq__(as_datetime)) different_day = (as_date.day + 1) % 20 + 1 self.assertFalse(as_date.__eq__(as_datetime.replace(day=different_day))) # And date should compare with other subclasses of date. If a # subclass wants to stop this, it's up to the subclass to do so. date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) self.assertEqual(as_date, date_sc) self.assertEqual(date_sc, as_date) # Ditto for datetimes. datetime_sc = SubclassDatetime(as_datetime.year, as_datetime.month, as_date.day, 0, 0, 0) self.assertEqual(as_datetime, datetime_sc) self.assertEqual(datetime_sc, as_datetime) def test_main(): test_support.run_unittest(__name__) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_dict.py ================================================ import unittest from test import test_support import UserDict, random, string # import gc, weakref class DictTest(unittest.TestCase): def test_constructor(self): # calling built-in types without argument must return empty self.assertEqual(dict(), {}) self.assertIsNot(dict(), {}) @unittest.expectedFailure def test_literal_constructor(self): # check literal constructor for different sized dicts # (to exercise the BUILD_MAP oparg). for n in (0, 1, 6, 256, 400): items = [(''.join(random.sample(string.letters, 8)), i) for i in range(n)] random.shuffle(items) formatted_items = ('{!r}: {:d}'.format(k, v) for k, v in items) dictliteral = '{' + ', '.join(formatted_items) + '}' self.assertEqual(eval(dictliteral), dict(items)) def test_bool(self): self.assertIs(not {}, True) self.assertTrue({1: 2}) self.assertIs(bool({}), False) self.assertIs(bool({1: 2}), True) @unittest.expectedFailure def test_keys(self): d = {} self.assertEqual(d.keys(), []) d = {'a': 1, 'b': 2} k = d.keys() # self.assertEqual(set(k), {'a', 'b'}) self.assertIn('a', k) self.assertIn('b', k) self.assertTrue(d.has_key('a')) self.assertTrue(d.has_key('b')) self.assertRaises(TypeError, d.keys, None) def test_values(self): d = {} self.assertEqual(d.values(), []) d = {1:2} self.assertEqual(d.values(), [2]) self.assertRaises(TypeError, d.values, None) def test_items(self): d = {} self.assertEqual(d.items(), []) d = {1:2} self.assertEqual(d.items(), [(1, 2)]) self.assertRaises(TypeError, d.items, None) @unittest.expectedFailure def test_has_key(self): d = {} self.assertFalse(d.has_key('a')) d = {'a': 1, 'b': 2} k = d.keys() k.sort() self.assertEqual(k, ['a', 'b']) self.assertRaises(TypeError, d.has_key) def test_contains(self): d = {} self.assertNotIn('a', d) self.assertFalse('a' in d) self.assertTrue('a' not in d) d = {'a': 1, 'b': 2} self.assertIn('a', d) self.assertIn('b', d) self.assertNotIn('c', d) self.assertRaises(TypeError, d.__contains__) def test_len(self): d = {} self.assertEqual(len(d), 0) d = {'a': 1, 'b': 2} self.assertEqual(len(d), 2) def test_getitem(self): d = {'a': 1, 'b': 2} self.assertEqual(d['a'], 1) self.assertEqual(d['b'], 2) d['c'] = 3 d['a'] = 4 self.assertEqual(d['c'], 3) self.assertEqual(d['a'], 4) del d['b'] self.assertEqual(d, {'a': 4, 'c': 3}) self.assertRaises(TypeError, d.__getitem__) class BadEq(object): def __eq__(self, other): raise Exc() def __hash__(self): return 24 d = {} d[BadEq()] = 42 self.assertRaises(KeyError, d.__getitem__, 23) class Exc(Exception): pass class BadHash(object): fail = False def __hash__(self): if self.fail: raise Exc() else: return 42 x = BadHash() d[x] = 42 x.fail = True self.assertRaises(Exc, d.__getitem__, x) def test_clear(self): d = {1:1, 2:2, 3:3} d.clear() self.assertEqual(d, {}) self.assertRaises(TypeError, d.clear, None) @unittest.expectedFailure def test_update(self): d = {} d.update({1:100}) d.update({2:20}) d.update({1:1, 2:2, 3:3}) self.assertEqual(d, {1:1, 2:2, 3:3}) d.update() self.assertEqual(d, {1:1, 2:2, 3:3}) self.assertRaises((TypeError, AttributeError), d.update, None) class SimpleUserDict(object): def __init__(self): self.d = {1:1, 2:2, 3:3} def keys(self): return self.d.keys() def __getitem__(self, i): return self.d[i] d.clear() d.update(SimpleUserDict()) self.assertEqual(d, {1:1, 2:2, 3:3}) class Exc(Exception): pass d.clear() class FailingUserDict(object): def keys(self): raise Exc self.assertRaises(Exc, d.update, FailingUserDict()) class FailingUserDict(object): def keys(self): class BogonIter(object): def __init__(self): self.i = 1 def __iter__(self): return self def next(self): if self.i: self.i = 0 return 'a' raise Exc return BogonIter() def __getitem__(self, key): return key self.assertRaises(Exc, d.update, FailingUserDict()) class FailingUserDict(object): def keys(self): class BogonIter(object): def __init__(self): self.i = ord('a') def __iter__(self): return self def next(self): if self.i <= ord('z'): rtn = chr(self.i) self.i += 1 return rtn raise StopIteration return BogonIter() def __getitem__(self, key): raise Exc self.assertRaises(Exc, d.update, FailingUserDict()) class badseq(object): def __iter__(self): return self def next(self): raise Exc() self.assertRaises(Exc, {}.update, badseq()) self.assertRaises(ValueError, {}.update, [(1, 2, 3)]) @unittest.expectedFailure def test_fromkeys(self): self.assertEqual(dict.fromkeys('abc'), {'a':None, 'b':None, 'c':None}) d = {} self.assertIsNot(d.fromkeys('abc'), d) self.assertEqual(d.fromkeys('abc'), {'a':None, 'b':None, 'c':None}) self.assertEqual(d.fromkeys((4,5),0), {4:0, 5:0}) self.assertEqual(d.fromkeys([]), {}) def g(): yield 1 self.assertEqual(d.fromkeys(g()), {1:None}) self.assertRaises(TypeError, {}.fromkeys, 3) class dictlike(dict): pass self.assertEqual(dictlike.fromkeys('a'), {'a':None}) self.assertEqual(dictlike().fromkeys('a'), {'a':None}) self.assertIsInstance(dictlike.fromkeys('a'), dictlike) self.assertIsInstance(dictlike().fromkeys('a'), dictlike) class mydict(dict): def __new__(cls): return UserDict.UserDict() ud = mydict.fromkeys('ab') self.assertEqual(ud, {'a':None, 'b':None}) self.assertIsInstance(ud, UserDict.UserDict) self.assertRaises(TypeError, dict.fromkeys) class Exc(Exception): pass class baddict1(dict): def __init__(self): raise Exc() self.assertRaises(Exc, baddict1.fromkeys, [1]) class BadSeq(object): def __iter__(self): return self def next(self): raise Exc() self.assertRaises(Exc, dict.fromkeys, BadSeq()) class baddict2(dict): def __setitem__(self, key, value): raise Exc() self.assertRaises(Exc, baddict2.fromkeys, [1]) # test fast path for dictionary inputs d = dict(zip(range(6), range(6))) self.assertEqual(dict.fromkeys(d, 0), dict(zip(range(6), [0]*6))) class baddict3(dict): def __new__(cls): return d d = {i : i for i in range(10)} res = d.copy() res.update(a=None, b=None, c=None) # self.assertEqual(baddict3.fromkeys({"a", "b", "c"}), res) @unittest.expectedFailure def test_copy(self): d = {1:1, 2:2, 3:3} self.assertEqual(d.copy(), {1:1, 2:2, 3:3}) self.assertEqual({}.copy(), {}) self.assertRaises(TypeError, d.copy, None) def test_get(self): d = {} self.assertIs(d.get('c'), None) self.assertEqual(d.get('c', 3), 3) d = {'a': 1, 'b': 2} self.assertIs(d.get('c'), None) self.assertEqual(d.get('c', 3), 3) self.assertEqual(d.get('a'), 1) self.assertEqual(d.get('a', 3), 1) self.assertRaises(TypeError, d.get) self.assertRaises(TypeError, d.get, None, None, None) def test_setdefault(self): # dict.setdefault() d = {} self.assertIs(d.setdefault('key0'), None) d.setdefault('key0', []) self.assertIs(d.setdefault('key0'), None) d.setdefault('key', []).append(3) self.assertEqual(d['key'][0], 3) d.setdefault('key', []).append(4) self.assertEqual(len(d['key']), 2) self.assertRaises(TypeError, d.setdefault) class Exc(Exception): pass class BadHash(object): fail = False def __hash__(self): if self.fail: raise Exc() else: return 42 x = BadHash() d[x] = 42 x.fail = True self.assertRaises(Exc, d.setdefault, x, []) def test_setdefault_atomic(self): # Issue #13521: setdefault() calls __hash__ and __eq__ only once. class Hashed(object): def __init__(self): self.hash_count = 0 self.eq_count = 0 def __hash__(self): self.hash_count += 1 return 42 def __eq__(self, other): self.eq_count += 1 return id(self) == id(other) hashed1 = Hashed() y = {hashed1: 5} hashed2 = Hashed() y.setdefault(hashed2, []) self.assertEqual(hashed1.hash_count, 1) self.assertEqual(hashed2.hash_count, 1) self.assertEqual(hashed1.eq_count + hashed2.eq_count, 1) @unittest.expectedFailure def test_popitem(self): # dict.popitem() # for copymode in -1, +1: for copymode in -1, 1: # -1: b has same structure as a # +1: b is a.copy() for log2size in range(12): size = 2**log2size a = {} b = {} for i in range(size): a[repr(i)] = i if copymode < 0: b[repr(i)] = i if copymode > 0: b = a.copy() for i in range(size): ka, va = ta = a.popitem() self.assertEqual(va, int(ka)) kb, vb = tb = b.popitem() self.assertEqual(vb, int(kb)) self.assertFalse(copymode < 0 and ta != tb) self.assertFalse(a) self.assertFalse(b) d = {} self.assertRaises(KeyError, d.popitem) @unittest.expectedFailure def test_pop(self): # Tests for pop with specified key d = {} k, v = 'abc', 'def' d[k] = v self.assertRaises(KeyError, d.pop, 'ghi') self.assertEqual(d.pop(k), v) self.assertEqual(len(d), 0) self.assertRaises(KeyError, d.pop, k) # verify longs/ints get same value when key > 32 bits # (for 64-bit archs). See SF bug #689659. x = 4503599627370496L y = 4503599627370496 h = {x: 'anything', y: 'something else'} self.assertEqual(h[x], h[y]) self.assertEqual(d.pop(k, v), v) d[k] = v self.assertEqual(d.pop(k, 1), v) self.assertRaises(TypeError, d.pop) class Exc(Exception): pass class BadHash(object): fail = False def __hash__(self): if self.fail: raise Exc() else: return 42 x = BadHash() d[x] = 42 x.fail = True self.assertRaises(Exc, d.pop, x) def test_mutatingiteration(self): # changing dict size during iteration d = {} d[1] = 1 with self.assertRaises(RuntimeError): for i in d: d[i+1] = 1 def test_repr(self): d = {} self.assertEqual(repr(d), '{}') d[1] = 2 self.assertEqual(repr(d), '{1: 2}') d = {} d[1] = d self.assertEqual(repr(d), '{1: {...}}') class Exc(Exception): pass class BadRepr(object): def __repr__(self): raise Exc() d = {1: BadRepr()} self.assertRaises(Exc, repr, d) @unittest.expectedFailure def test_le(self): self.assertFalse({} < {}) self.assertFalse({1: 2} < {1L: 2L}) class Exc(Exception): pass class BadCmp(object): def __eq__(self, other): raise Exc() def __hash__(self): return 42 d1 = {BadCmp(): 1} d2 = {1: 1} with self.assertRaises(Exc): d1 < d2 @unittest.expectedFailure def test_missing(self): # Make sure dict doesn't have a __missing__ method self.assertFalse(hasattr(dict, "__missing__")) self.assertFalse(hasattr({}, "__missing__")) # Test several cases: # (D) subclass defines __missing__ method returning a value # (E) subclass defines __missing__ method raising RuntimeError # (F) subclass sets __missing__ instance variable (no effect) # (G) subclass doesn't define __missing__ at all class D(dict): def __missing__(self, key): return 42 d = D({1: 2, 3: 4}) self.assertEqual(d[1], 2) self.assertEqual(d[3], 4) self.assertNotIn(2, d) self.assertNotIn(2, d.keys()) self.assertEqual(d[2], 42) class E(dict): def __missing__(self, key): raise RuntimeError(key) e = E() with self.assertRaises(RuntimeError) as c: e[42] self.assertEqual(c.exception.args, (42,)) class F(dict): def __init__(self): # An instance variable __missing__ should have no effect self.__missing__ = lambda key: None f = F() with self.assertRaises(KeyError) as c: f[42] self.assertEqual(c.exception.args, (42,)) class G(dict): pass g = G() with self.assertRaises(KeyError) as c: g[42] self.assertEqual(c.exception.args, (42,)) @unittest.expectedFailure def test_tuple_keyerror(self): # SF #1576657 d = {} with self.assertRaises(KeyError) as c: d[(1,)] self.assertEqual(c.exception.args, ((1,),)) # def test_bad_key(self): # # Dictionary lookups should fail if __cmp__() raises an exception. # class CustomException(Exception): # pass # class BadDictKey(object): # def __hash__(self): # return hash(self.__class__) # def __cmp__(self, other): # if isinstance(other, self.__class__): # raise CustomException # return other # d = {} # x1 = BadDictKey() # x2 = BadDictKey() # d[x1] = 1 # for stmt in ['d[x2] = 2', # 'z = d[x2]', # 'x2 in d', # 'd.has_key(x2)', # 'd.get(x2)', # 'd.setdefault(x2, 42)', # 'd.pop(x2)', # 'd.update({x2: 2})']: # with self.assertRaises(CustomException): # exec stmt in locals() def test_resize1(self): # Dict resizing bug, found by Jack Jansen in 2.2 CVS development. # This version got an assert failure in debug build, infinite loop in # release build. Unfortunately, provoking this kind of stuff requires # a mix of inserts and deletes hitting exactly the right hash codes in # exactly the right order, and I can't think of a randomized approach # that would be *likely* to hit a failing case in reasonable time. d = {} for i in range(5): d[i] = i for i in range(5): del d[i] for i in range(5, 9): # i==8 was the problem d[i] = i def test_resize2(self): # Another dict resizing bug (SF bug #1456209). # This caused Segmentation faults or Illegal instructions. class X(object): def __hash__(self): return 5 def __eq__(self, other): if resizing: d.clear() return False d = {} resizing = False d[X()] = 1 d[X()] = 2 d[X()] = 3 d[X()] = 4 d[X()] = 5 # now trigger a resize resizing = True d[9] = 6 def test_empty_presized_dict_in_freelist(self): # Bug #3537: if an empty but presized dict with a size larger # than 7 was in the freelist, it triggered an assertion failure with self.assertRaises(ZeroDivisionError): d = {'a': 1 // 0, 'b': None, 'c': None, 'd': None, 'e': None, 'f': None, 'g': None, 'h': None} d = {} # def test_container_iterator(self): # # Bug #3680: tp_traverse was not implemented for dictiter objects # class C(object): # pass # iterators = (dict.iteritems, dict.itervalues, dict.iterkeys) # for i in iterators: # obj = C() # ref = weakref.ref(obj) # container = {obj: 1} # obj.x = i(container) # del obj, container # gc.collect() # self.assertIs(ref(), None, "Cycle was not collected") # def _not_tracked(self, t): # # Nested containers can take several collections to untrack # gc.collect() # gc.collect() # self.assertFalse(gc.is_tracked(t), t) # def _tracked(self, t): # self.assertTrue(gc.is_tracked(t), t) # gc.collect() # gc.collect() # self.assertTrue(gc.is_tracked(t), t) @test_support.cpython_only def test_track_literals(self): # Test GC-optimization of dict literals x, y, z, w = 1.5, "a", (1, None), [] self._not_tracked({}) self._not_tracked({x:(), y:x, z:1}) self._not_tracked({1: "a", "b": 2}) self._not_tracked({1: 2, (None, True, False, ()): int}) self._not_tracked({1: object()}) # Dicts with mutable elements are always tracked, even if those # elements are not tracked right now. self._tracked({1: []}) self._tracked({1: ([],)}) self._tracked({1: {}}) self._tracked({1: set()}) @test_support.cpython_only def test_track_dynamic(self): # Test GC-optimization of dynamically-created dicts class MyObject(object): pass x, y, z, w, o = 1.5, "a", (1, object()), [], MyObject() d = dict() self._not_tracked(d) d[1] = "a" self._not_tracked(d) d[y] = 2 self._not_tracked(d) d[z] = 3 self._not_tracked(d) self._not_tracked(d.copy()) d[4] = w self._tracked(d) self._tracked(d.copy()) d[4] = None self._not_tracked(d) self._not_tracked(d.copy()) # dd isn't tracked right now, but it may mutate and therefore d # which contains it must be tracked. d = dict() dd = dict() d[1] = dd self._not_tracked(dd) self._tracked(d) dd[1] = d self._tracked(dd) d = dict.fromkeys([x, y, z]) self._not_tracked(d) dd = dict() dd.update(d) self._not_tracked(dd) d = dict.fromkeys([x, y, z, o]) self._tracked(d) dd = dict() dd.update(d) self._tracked(dd) d = dict(x=x, y=y, z=z) self._not_tracked(d) d = dict(x=x, y=y, z=z, w=w) self._tracked(d) d = dict() d.update(x=x, y=y, z=z) self._not_tracked(d) d.update(w=w) self._tracked(d) d = dict([(x, y), (z, 1)]) self._not_tracked(d) d = dict([(x, y), (z, w)]) self._tracked(d) d = dict() d.update([(x, y), (z, 1)]) self._not_tracked(d) d.update([(x, y), (z, w)]) self._tracked(d) @test_support.cpython_only def test_track_subtypes(self): # Dict subtypes are always tracked class MyDict(dict): pass self._tracked(MyDict()) # def test_free_after_iterating(self): # test_support.check_free_after_iterating(self, iter, dict) # test_support.check_free_after_iterating(self, lambda d: d.iterkeys(), dict) # test_support.check_free_after_iterating(self, lambda d: d.itervalues(), dict) # test_support.check_free_after_iterating(self, lambda d: d.iteritems(), dict) # test_support.check_free_after_iterating(self, lambda d: iter(d.viewkeys()), dict) # test_support.check_free_after_iterating(self, lambda d: iter(d.viewvalues()), dict) # test_support.check_free_after_iterating(self, lambda d: iter(d.viewitems()), dict) from test import mapping_tests class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol): type2test = dict class Dict(dict): pass class SubclassMappingTests(mapping_tests.BasicTestMappingProtocol): type2test = Dict def test_main(): with test_support.check_py3k_warnings( ('dict(.has_key..| inequality comparisons) not supported in 3.x', DeprecationWarning)): test_support.run_unittest( DictTest, GeneralMappingTests, SubclassMappingTests, ) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_dircache.py ================================================ """ Test cases for the dircache module Nick Mathewson """ import unittest from test.test_support import run_unittest # , import_module # dircache = import_module('dircache', deprecated=True) import dircache import os, time, sys, tempfile class DircacheTests(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() def tearDown(self): for fname in os.listdir(self.tempdir): self.delTemp(fname) os.rmdir(self.tempdir) def writeTemp(self, fname): f = open(os.path.join(self.tempdir, fname), 'w') f.close() def mkdirTemp(self, fname): os.mkdir(os.path.join(self.tempdir, fname)) def delTemp(self, fname): fname = os.path.join(self.tempdir, fname) if os.path.isdir(fname): os.rmdir(fname) else: os.unlink(fname) def test_listdir(self): ## SUCCESSFUL CASES entries = dircache.listdir(self.tempdir) self.assertEqual(entries, []) # Check that cache is actually caching, not just passing through. self.assertTrue(dircache.listdir(self.tempdir) is entries) # Directories aren't "files" on Windows, and directory mtime has # nothing to do with when files under a directory get created. # That is, this test can't possibly work under Windows -- dircache # is only good for capturing a one-shot snapshot there. if sys.platform[:3] not in ('win', 'os2'): # Sadly, dircache has the same granularity as stat.mtime, and so # can't notice any changes that occurred within 1 sec of the last # time it examined a directory. time.sleep(1) self.writeTemp("test1") entries = dircache.listdir(self.tempdir) self.assertEqual(entries, ['test1']) self.assertTrue(dircache.listdir(self.tempdir) is entries) ## UNSUCCESSFUL CASES self.assertRaises(OSError, dircache.listdir, self.tempdir+"_nonexistent") def test_annotate(self): self.writeTemp("test2") self.mkdirTemp("A") lst = ['A', 'test2', 'test_nonexistent'] dircache.annotate(self.tempdir, lst) self.assertEqual(lst, ['A/', 'test2', 'test_nonexistent']) def test_main(): try: run_unittest(DircacheTests) finally: dircache.reset() if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_dummy_thread.py ================================================ """Generic thread tests. Meant to be used by dummy_thread and thread. To allow for different modules to be used, test_main() can be called with the module to use as the thread implementation as its sole argument. """ import dummy_thread as _thread import time import Queue import random import unittest from test import test_support DELAY = 0 # Set > 0 when testing a module other than dummy_thread, such as # the 'thread' module. class LockTests(unittest.TestCase): """Test lock objects.""" def setUp(self): # Create a lock self.lock = _thread.allocate_lock() def test_initlock(self): #Make sure locks start locked self.assertFalse(self.lock.locked(), "Lock object is not initialized unlocked.") def test_release(self): # Test self.lock.release() self.lock.acquire() self.lock.release() self.assertFalse(self.lock.locked(), "Lock object did not release properly.") def test_improper_release(self): #Make sure release of an unlocked thread raises _thread.error self.assertRaises(_thread.error, self.lock.release) def test_cond_acquire_success(self): #Make sure the conditional acquiring of the lock works. self.assertTrue(self.lock.acquire(0), "Conditional acquiring of the lock failed.") def test_cond_acquire_fail(self): #Test acquiring locked lock returns False self.lock.acquire(0) self.assertFalse(self.lock.acquire(0), "Conditional acquiring of a locked lock incorrectly " "succeeded.") def test_uncond_acquire_success(self): #Make sure unconditional acquiring of a lock works. self.lock.acquire() self.assertTrue(self.lock.locked(), "Uncondional locking failed.") def test_uncond_acquire_return_val(self): #Make sure that an unconditional locking returns True. self.assertIs(self.lock.acquire(1), True, "Unconditional locking did not return True.") self.assertIs(self.lock.acquire(), True) def test_uncond_acquire_blocking(self): #Make sure that unconditional acquiring of a locked lock blocks. def delay_unlock(to_unlock, delay): """Hold on to lock for a set amount of time before unlocking.""" time.sleep(delay) to_unlock.release() self.lock.acquire() start_time = int(time.time()) _thread.start_new_thread(delay_unlock,(self.lock, DELAY)) if test_support.verbose: print print "*** Waiting for thread to release the lock "\ "(approx. %s sec.) ***" % DELAY self.lock.acquire() end_time = int(time.time()) if test_support.verbose: print "done" self.assertGreaterEqual(end_time - start_time, DELAY, "Blocking by unconditional acquiring failed.") class MiscTests(unittest.TestCase): """Miscellaneous tests.""" def test_exit(self): #Make sure _thread.exit() raises SystemExit self.assertRaises(SystemExit, _thread.exit) def test_ident(self): #Test sanity of _thread.get_ident() self.assertIsInstance(_thread.get_ident(), int, "_thread.get_ident() returned a non-integer") self.assertNotEqual(_thread.get_ident(), 0, "_thread.get_ident() returned 0") def test_LockType(self): #Make sure _thread.LockType is the same type as _thread.allocate_locke() self.assertIsInstance(_thread.allocate_lock(), _thread.LockType, "_thread.LockType is not an instance of what " "is returned by _thread.allocate_lock()") def test_interrupt_main(self): #Calling start_new_thread with a function that executes interrupt_main # should raise KeyboardInterrupt upon completion. def call_interrupt(): _thread.interrupt_main() self.assertRaises(KeyboardInterrupt, _thread.start_new_thread, call_interrupt, tuple()) def test_interrupt_in_main(self): # Make sure that if interrupt_main is called in main threat that # KeyboardInterrupt is raised instantly. self.assertRaises(KeyboardInterrupt, _thread.interrupt_main) class ThreadTests(unittest.TestCase): """Test thread creation.""" def test_arg_passing(self): #Make sure that parameter passing works. def arg_tester(queue, arg1=False, arg2=False): """Use to test _thread.start_new_thread() passes args properly.""" queue.put((arg1, arg2)) testing_queue = Queue.Queue(1) _thread.start_new_thread(arg_tester, (testing_queue, True, True)) result = testing_queue.get() self.assertTrue(result[0] and result[1], "Argument passing for thread creation using tuple failed") _thread.start_new_thread(arg_tester, tuple(), {'queue':testing_queue, 'arg1':True, 'arg2':True}) result = testing_queue.get() self.assertTrue(result[0] and result[1], "Argument passing for thread creation using kwargs failed") _thread.start_new_thread(arg_tester, (testing_queue, True), {'arg2':True}) result = testing_queue.get() self.assertTrue(result[0] and result[1], "Argument passing for thread creation using both tuple" " and kwargs failed") def test_multi_creation(self): #Make sure multiple threads can be created. def queue_mark(queue, delay): """Wait for ``delay`` seconds and then put something into ``queue``""" time.sleep(delay) queue.put(_thread.get_ident()) thread_count = 5 testing_queue = Queue.Queue(thread_count) if test_support.verbose: print print "*** Testing multiple thread creation "\ "(will take approx. %s to %s sec.) ***" % (DELAY, thread_count) for count in xrange(thread_count): if DELAY: local_delay = round(random.random(), 1) else: local_delay = 0 _thread.start_new_thread(queue_mark, (testing_queue, local_delay)) time.sleep(DELAY) if test_support.verbose: print 'done' self.assertEqual(testing_queue.qsize(), thread_count, "Not all %s threads executed properly after %s sec." % (thread_count, DELAY)) def test_main(imported_module=None): global _thread, DELAY if imported_module: _thread = imported_module DELAY = 2 if test_support.verbose: print print "*** Using %s as _thread module ***" % _thread test_support.run_unittest(LockTests, MiscTests, ThreadTests) if __name__ == '__main__': test_main() ================================================ FILE: third_party/stdlib/test/test_fpformat.py ================================================ ''' Tests for fpformat module Nick Mathewson ''' from test.test_support import run_unittest #, import_module import unittest # fpformat = import_module('fpformat', deprecated=True) import fpformat fix, sci, NotANumber = fpformat.fix, fpformat.sci, fpformat.NotANumber StringType = type('') # Test the old and obsolescent fpformat module. # # (It's obsolescent because fix(n,d) == "%.*f"%(d,n) and # sci(n,d) == "%.*e"%(d,n) # for all reasonable numeric n and d, except that sci gives 3 exponent # digits instead of 2. # # Differences only occur for unreasonable n and d. <.2 wink>) class FpformatTest(unittest.TestCase): def checkFix(self, n, digits): result = fix(n, digits) if isinstance(n, StringType): n = repr(n) expected = "%.*f" % (digits, float(n)) self.assertEqual(result, expected) def checkSci(self, n, digits): result = sci(n, digits) if isinstance(n, StringType): n = repr(n) expected = "%.*e" % (digits, float(n)) # add the extra 0 if needed num, exp = expected.split("e") if len(exp) < 4: exp = exp[0] + "0" + exp[1:] expected = "%se%s" % (num, exp) self.assertEqual(result, expected) def test_basic_cases(self): self.assertEqual(fix(100.0/3, 3), '33.333') self.assertEqual(sci(100.0/3, 3), '3.333e+001') @unittest.skip('grumpy') def test_reasonable_values(self): for d in range(7): for val in (1000.0/3, 1000, 1000.0, .002, 1.0/3, 1e10): for realVal in (val, 1.0/val, -val, -1.0/val): self.checkFix(realVal, d) self.checkSci(realVal, d) def test_failing_values(self): # Now for 'unreasonable n and d' self.assertEqual(fix(1.0, 1000), '1.'+('0'*1000)) self.assertEqual(sci("1"+('0'*1000), 0), '1e+1000') # This behavior is inconsistent. sci raises an exception; fix doesn't. yacht = "Throatwobbler Mangrove" self.assertEqual(fix(yacht, 10), yacht) try: sci(yacht, 10) except NotANumber: pass else: self.fail("No exception on non-numeric sci") def test_main(): run_unittest(FpformatTest) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_genericpath.py ================================================ """ Tests common to genericpath, macpath, ntpath and posixpath """ import unittest from test import test_support import os import genericpath import sys def safe_rmdir(dirname): try: os.rmdir(dirname) except OSError: pass class GenericTest(unittest.TestCase): # The path module to be tested pathmodule = genericpath common_attributes = ['commonprefix', 'getsize', 'getatime', 'getctime', 'getmtime', 'exists', 'isdir', 'isfile'] attributes = [] def test_no_argument(self): for attr in self.common_attributes + self.attributes: with self.assertRaises(TypeError): getattr(self.pathmodule, attr)() raise self.fail("{}.{}() did not raise a TypeError" .format(self.pathmodule.__name__, attr)) def test_commonprefix(self): commonprefix = self.pathmodule.commonprefix self.assertEqual( commonprefix([]), "" ) self.assertEqual( commonprefix(["/home/swenson/spam", "/home/swen/spam"]), "/home/swen" ) self.assertEqual( commonprefix(["/home/swen/spam", "/home/swen/eggs"]), "/home/swen/" ) self.assertEqual( commonprefix(["/home/swen/spam", "/home/swen/spam"]), "/home/swen/spam" ) self.assertEqual( commonprefix(["home:swenson:spam", "home:swen:spam"]), "home:swen" ) self.assertEqual( commonprefix([":home:swen:spam", ":home:swen:eggs"]), ":home:swen:" ) self.assertEqual( commonprefix([":home:swen:spam", ":home:swen:spam"]), ":home:swen:spam" ) testlist = ['', 'abc', 'Xbcd', 'Xb', 'XY', 'abcd', 'aXc', 'abd', 'ab', 'aX', 'abcX'] for s1 in testlist: for s2 in testlist: p = commonprefix([s1, s2]) self.assertTrue(s1.startswith(p)) self.assertTrue(s2.startswith(p)) if s1 != s2: n = len(p) self.assertNotEqual(s1[n:n+1], s2[n:n+1]) def test_getsize(self): f = open(test_support.TESTFN, "wb") try: f.write("foo") f.close() self.assertEqual(self.pathmodule.getsize(test_support.TESTFN), 3) finally: if not f.closed: f.close() test_support.unlink(test_support.TESTFN) @unittest.skip('grumpy') def test_time(self): f = open(test_support.TESTFN, "wb") try: f.write("foo") f.close() f = open(test_support.TESTFN, "ab") f.write("bar") f.close() f = open(test_support.TESTFN, "rb") d = f.read() f.close() self.assertEqual(d, "foobar") self.assertLessEqual( self.pathmodule.getctime(test_support.TESTFN), self.pathmodule.getmtime(test_support.TESTFN) ) finally: if not f.closed: f.close() test_support.unlink(test_support.TESTFN) def test_exists(self): self.assertIs(self.pathmodule.exists(test_support.TESTFN), False) f = open(test_support.TESTFN, "wb") try: f.write("foo") f.close() self.assertIs(self.pathmodule.exists(test_support.TESTFN), True) if not self.pathmodule == genericpath: self.assertIs(self.pathmodule.lexists(test_support.TESTFN), True) finally: if not f.close(): f.close() test_support.unlink(test_support.TESTFN) def test_isdir(self): self.assertIs(self.pathmodule.isdir(test_support.TESTFN), False) f = open(test_support.TESTFN, "wb") try: f.write("foo") f.close() self.assertIs(self.pathmodule.isdir(test_support.TESTFN), False) os.remove(test_support.TESTFN) os.mkdir(test_support.TESTFN) self.assertIs(self.pathmodule.isdir(test_support.TESTFN), True) os.rmdir(test_support.TESTFN) finally: if not f.close(): f.close() test_support.unlink(test_support.TESTFN) safe_rmdir(test_support.TESTFN) @unittest.skip('grumpy') def test_isfile(self): self.assertIs(self.pathmodule.isfile(test_support.TESTFN), False) f = open(test_support.TESTFN, "wb") try: f.write("foo") f.close() self.assertIs(self.pathmodule.isfile(test_support.TESTFN), True) os.remove(test_support.TESTFN) os.mkdir(test_support.TESTFN) self.assertIs(self.pathmodule.isfile(test_support.TESTFN), False) os.rmdir(test_support.TESTFN) finally: if not f.close(): f.close() test_support.unlink(test_support.TESTFN) safe_rmdir(test_support.TESTFN) # Following TestCase is not supposed to be run from test_genericpath. # It is inherited by other test modules (macpath, ntpath, posixpath). class CommonTest(GenericTest): # The path module to be tested pathmodule = None common_attributes = GenericTest.common_attributes + [ # Properties 'curdir', 'pardir', 'extsep', 'sep', 'pathsep', 'defpath', 'altsep', 'devnull', # Methods 'normcase', 'splitdrive', 'expandvars', 'normpath', 'abspath', 'join', 'split', 'splitext', 'isabs', 'basename', 'dirname', 'lexists', 'islink', 'ismount', 'expanduser', 'normpath', 'realpath', ] def test_normcase(self): # Check that normcase() is idempotent p = "FoO/./BaR" p = self.pathmodule.normcase(p) self.assertEqual(p, self.pathmodule.normcase(p)) def test_splitdrive(self): # splitdrive for non-NT paths splitdrive = self.pathmodule.splitdrive self.assertEqual(splitdrive("/foo/bar"), ("", "/foo/bar")) self.assertEqual(splitdrive("foo:bar"), ("", "foo:bar")) self.assertEqual(splitdrive(":foo:bar"), ("", ":foo:bar")) def test_expandvars(self): if self.pathmodule.__name__ == 'macpath': self.skipTest('macpath.expandvars is a stub') expandvars = self.pathmodule.expandvars with test_support.EnvironmentVarGuard() as env: env.clear() env["foo"] = "bar" env["{foo"] = "baz1" env["{foo}"] = "baz2" self.assertEqual(expandvars("foo"), "foo") self.assertEqual(expandvars("$foo bar"), "bar bar") self.assertEqual(expandvars("${foo}bar"), "barbar") self.assertEqual(expandvars("$[foo]bar"), "$[foo]bar") self.assertEqual(expandvars("$bar bar"), "$bar bar") self.assertEqual(expandvars("$?bar"), "$?bar") self.assertEqual(expandvars("$foo}bar"), "bar}bar") self.assertEqual(expandvars("${foo"), "${foo") self.assertEqual(expandvars("${{foo}}"), "baz1}") self.assertEqual(expandvars("$foo$foo"), "barbar") self.assertEqual(expandvars("$bar$bar"), "$bar$bar") @unittest.skipUnless(test_support.FS_NONASCII, 'need test_support.FS_NONASCII') def test_expandvars_nonascii(self): if self.pathmodule.__name__ == 'macpath': self.skipTest('macpath.expandvars is a stub') expandvars = self.pathmodule.expandvars def check(value, expected): self.assertEqual(expandvars(value), expected) encoding = sys.getfilesystemencoding() with test_support.EnvironmentVarGuard() as env: env.clear() unonascii = test_support.FS_NONASCII snonascii = unonascii.encode(encoding) env['spam'] = snonascii env[snonascii] = 'ham' + snonascii check(snonascii, snonascii) check('$spam bar', '%s bar' % snonascii) check('${spam}bar', '%sbar' % snonascii) check('${%s}bar' % snonascii, 'ham%sbar' % snonascii) check('$bar%s bar' % snonascii, '$bar%s bar' % snonascii) check('$spam}bar', '%s}bar' % snonascii) check(unonascii, unonascii) check(u'$spam bar', u'%s bar' % unonascii) check(u'${spam}bar', u'%sbar' % unonascii) check(u'${%s}bar' % unonascii, u'ham%sbar' % unonascii) check(u'$bar%s bar' % unonascii, u'$bar%s bar' % unonascii) check(u'$spam}bar', u'%s}bar' % unonascii) def test_abspath(self): self.assertIn("foo", self.pathmodule.abspath("foo")) # Abspath returns bytes when the arg is bytes for path in ('', 'foo', 'f\xf2\xf2', '/foo', 'C:\\'): self.assertIsInstance(self.pathmodule.abspath(path), str) def test_realpath(self): self.assertIn("foo", self.pathmodule.realpath("foo")) @test_support.requires_unicode def test_normpath_issue5827(self): # Make sure normpath preserves unicode for path in (u'', u'.', u'/', u'\\', u'///foo/.//bar//'): self.assertIsInstance(self.pathmodule.normpath(path), unicode) @test_support.requires_unicode def test_abspath_issue3426(self): # Check that abspath returns unicode when the arg is unicode # with both ASCII and non-ASCII cwds. abspath = self.pathmodule.abspath for path in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\'): self.assertIsInstance(abspath(path), unicode) unicwd = u'\xe7w\xf0' try: fsencoding = test_support.TESTFN_ENCODING or "ascii" unicwd.encode(fsencoding) except (AttributeError, UnicodeEncodeError): # FS encoding is probably ASCII pass else: with test_support.temp_cwd(unicwd): for path in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\'): self.assertIsInstance(abspath(path), unicode) @unittest.skipIf(sys.platform == 'darwin', "Mac OS X denies the creation of a directory with an invalid utf8 name") def test_nonascii_abspath(self): # Test non-ASCII, non-UTF8 bytes in the path. with test_support.temp_cwd('\xe7w\xf0'): self.test_abspath() def test_main(): test_support.run_unittest(GenericTest) if __name__=="__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_list.py ================================================ import sys import unittest from test import test_support, list_tests class ListTest(list_tests.CommonTest): type2test = list def test_basic(self): self.assertEqual(list([]), []) l0_3 = [0, 1, 2, 3] l0_3_bis = list(l0_3) self.assertEqual(l0_3, l0_3_bis) self.assertTrue(l0_3 is not l0_3_bis) self.assertEqual(list(()), []) self.assertEqual(list((0, 1, 2, 3)), [0, 1, 2, 3]) self.assertEqual(list(''), []) self.assertEqual(list('spam'), ['s', 'p', 'a', 'm']) if sys.maxsize == 0x7fffffff: # This test can currently only work on 32-bit machines. # XXX If/when PySequence_Length() returns a ssize_t, it should be # XXX re-enabled. # Verify clearing of bug #556025. # This assumes that the max data size (sys.maxint) == max # address size this also assumes that the address size is at # least 4 bytes with 8 byte addresses, the bug is not well # tested # # Note: This test is expected to SEGV under Cygwin 1.3.12 or # earlier due to a newlib bug. See the following mailing list # thread for the details: # http://sources.redhat.com/ml/newlib/2002/msg00369.html self.assertRaises(MemoryError, list, xrange(sys.maxint // 2)) # This code used to segfault in Py2.4a3 x = [] x.extend(-y for y in x) self.assertEqual(x, []) def test_truth(self): super(ListTest, self).test_truth() self.assertTrue(not []) self.assertTrue([42]) def test_identity(self): self.assertTrue([] is not []) def test_len(self): super(ListTest, self).test_len() self.assertEqual(len([]), 0) self.assertEqual(len([0]), 1) self.assertEqual(len([0, 1, 2]), 3) @unittest.expectedFailure def test_overflow(self): lst = [4, 5, 6, 7] n = int((sys.maxsize*2+2) // len(lst)) def mul(a, b): return a * b def imul(a, b): a *= b self.assertRaises((MemoryError, OverflowError), mul, lst, n) self.assertRaises((MemoryError, OverflowError), imul, lst, n) def test_main(verbose=None): test_support.run_unittest(ListTest) # verify reference counting # import sys # if verbose and hasattr(sys, "gettotalrefcount"): # import gc # counts = [None] * 5 # for i in xrange(len(counts)): # test_support.run_unittest(ListTest) # gc.collect() # counts[i] = sys.gettotalrefcount() # print counts if __name__ == "__main__": test_main(verbose=True) ================================================ FILE: third_party/stdlib/test/test_md5.py ================================================ # Testing md5 module import warnings warnings.filterwarnings("ignore", "the md5 module is deprecated.*", DeprecationWarning) import unittest # from md5 import md5 import md5 as _md5 md5 = _md5.md5 from test import test_support def hexstr(s): import string h = string.hexdigits r = '' for c in s: i = ord(c) r = r + h[(i >> 4) & 0xF] + h[i & 0xF] return r class MD5_Test(unittest.TestCase): def md5test(self, s, expected): self.assertEqual(hexstr(md5(s).digest()), expected) self.assertEqual(md5(s).hexdigest(), expected) def test_basics(self): eq = self.md5test eq('', 'd41d8cd98f00b204e9800998ecf8427e') eq('a', '0cc175b9c0f1b6a831c399e269772661') eq('abc', '900150983cd24fb0d6963f7d28e17f72') eq('message digest', 'f96b697d7cb7938d525a2f31aaf161d0') eq('abcdefghijklmnopqrstuvwxyz', 'c3fcd3d76192e4007dfb496cca67e13b') eq('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', 'd174ab98d277d9f5a5611c2c9f419d9f') eq('12345678901234567890123456789012345678901234567890123456789012345678901234567890', '57edf4a22be3c955ac49da2e2107b67a') def test_hexdigest(self): # hexdigest is new with Python 2.0 m = md5('testing the hexdigest method') h = m.hexdigest() self.assertEqual(hexstr(m.digest()), h) def test_large_update(self): aas = 'a' * 64 bees = 'b' * 64 cees = 'c' * 64 m1 = md5() m1.update(aas) m1.update(bees) m1.update(cees) m2 = md5() m2.update(aas + bees + cees) self.assertEqual(m1.digest(), m2.digest()) def test_main(): test_support.run_unittest(MD5_Test) if __name__ == '__main__': test_main() ================================================ FILE: third_party/stdlib/test/test_mimetools.py ================================================ import unittest from test import test_support import string import StringIO #mimetools = test_support.import_module("mimetools", deprecated=True) import mimetools msgtext1 = mimetools.Message(StringIO.StringIO( """Content-Type: text/plain; charset=iso-8859-1; format=flowed Content-Transfer-Encoding: 8bit Foo! """)) class MimeToolsTest(unittest.TestCase): def test_decodeencode(self): start = string.ascii_letters + "=" + string.digits + "\n" for enc in ['7bit','8bit','base64','quoted-printable', 'uuencode', 'x-uuencode', 'uue', 'x-uue']: i = StringIO.StringIO(start) o = StringIO.StringIO() mimetools.encode(i, o, enc) i = StringIO.StringIO(o.getvalue()) o = StringIO.StringIO() mimetools.decode(i, o, enc) self.assertEqual(o.getvalue(), start) @unittest.expectedFailure def test_boundary(self): s = set([""]) for i in xrange(100): nb = mimetools.choose_boundary() self.assertNotIn(nb, s) s.add(nb) def test_message(self): msg = mimetools.Message(StringIO.StringIO(msgtext1)) self.assertEqual(msg.gettype(), "text/plain") self.assertEqual(msg.getmaintype(), "text") self.assertEqual(msg.getsubtype(), "plain") self.assertEqual(msg.getplist(), ["charset=iso-8859-1", "format=flowed"]) self.assertEqual(msg.getparamnames(), ["charset", "format"]) self.assertEqual(msg.getparam("charset"), "iso-8859-1") self.assertEqual(msg.getparam("format"), "flowed") self.assertEqual(msg.getparam("spam"), None) self.assertEqual(msg.getencoding(), "8bit") def test_main(): test_support.run_unittest(MimeToolsTest) if __name__=="__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_mutex.py ================================================ import unittest import test.test_support # mutex = test.test_support.import_module("mutex", deprecated=True) import mutex class MutexTest(unittest.TestCase): def test_lock_and_unlock(self): def called_by_mutex(some_data): self.assertEqual(some_data, "spam") self.assertTrue(m.test(), "mutex not held") # Nested locking m.lock(called_by_mutex2, "eggs") def called_by_mutex2(some_data): self.assertEqual(some_data, "eggs") self.assertTrue(m.test(), "mutex not held") self.assertTrue(ready_for_2, "called_by_mutex2 called too soon") m = mutex.mutex() read_for_2 = False m.lock(called_by_mutex, "spam") ready_for_2 = True # unlock both locks m.unlock() m.unlock() self.assertFalse(m.test(), "mutex still held") def test_main(): test.test_support.run_unittest(MutexTest) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_queue.py ================================================ # Some simple queue module tests, plus some failure conditions # to ensure the Queue locks remain stable. import Queue import time import unittest from test import test_support #threading = test_support.import_module('threading') import threading QUEUE_SIZE = 5 # A thread to run a function that unclogs a blocked Queue. class _TriggerThread(threading.Thread): def __init__(self, fn, args): self.fn = fn self.args = args self.startedEvent = threading.Event() threading.Thread.__init__(self) def run(self): # The sleep isn't necessary, but is intended to give the blocking # function in the main thread a chance at actually blocking before # we unclog it. But if the sleep is longer than the timeout-based # tests wait in their blocking functions, those tests will fail. # So we give them much longer timeout values compared to the # sleep here (I aimed at 10 seconds for blocking functions -- # they should never actually wait that long - they should make # progress as soon as we call self.fn()). time.sleep(0.1) self.startedEvent.set() self.fn(*self.args) # Execute a function that blocks, and in a separate thread, a function that # triggers the release. Returns the result of the blocking function. Caution: # block_func must guarantee to block until trigger_func is called, and # trigger_func must guarantee to change queue state so that block_func can make # enough progress to return. In particular, a block_func that just raises an # exception regardless of whether trigger_func is called will lead to # timing-dependent sporadic failures, and one of those went rarely seen but # undiagnosed for years. Now block_func must be unexceptional. If block_func # is supposed to raise an exception, call do_exceptional_blocking_test() # instead. class BlockingTestMixin(object): def tearDown(self): self.t = None def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): self.t = _TriggerThread(trigger_func, trigger_args) self.t.start() self.result = block_func(*block_args) # If block_func returned before our thread made the call, we failed! if not self.t.startedEvent.is_set(): self.fail("blocking function '%r' appeared not to block" % block_func) self.t.join(10) # make sure the thread terminates if self.t.is_alive(): self.fail("trigger function '%r' appeared to not return" % trigger_func) return self.result # Call this instead if block_func is supposed to raise an exception. def do_exceptional_blocking_test(self,block_func, block_args, trigger_func, trigger_args, expected_exception_class): self.t = _TriggerThread(trigger_func, trigger_args) self.t.start() try: try: block_func(*block_args) except expected_exception_class: raise else: self.fail("expected exception of kind %r" % expected_exception_class) finally: self.t.join(10) # make sure the thread terminates if self.t.is_alive(): self.fail("trigger function '%r' appeared to not return" % trigger_func) if not self.t.startedEvent.is_set(): self.fail("trigger thread ended but event never set") class BaseQueueTest(BlockingTestMixin): def setUp(self): self.cum = 0 self.cumlock = threading.Lock() def simple_queue_test(self, q): if not q.empty(): raise RuntimeError, "Call this function with an empty queue" # I guess we better check things actually queue correctly a little :) q.put(111) q.put(333) q.put(222) target_order = dict(Queue = [111, 333, 222], LifoQueue = [222, 333, 111], PriorityQueue = [111, 222, 333]) actual_order = [q.get(), q.get(), q.get()] self.assertEqual(actual_order, target_order[q.__class__.__name__], "Didn't seem to queue the correct data!") for i in range(QUEUE_SIZE-1): q.put(i) self.assertTrue(not q.empty(), "Queue should not be empty") self.assertTrue(not q.full(), "Queue should not be full") last = 2 * QUEUE_SIZE full = 3 * 2 * QUEUE_SIZE q.put(last) self.assertTrue(q.full(), "Queue should be full") try: q.put(full, block=0) self.fail("Didn't appear to block with a full queue") except Queue.Full: pass try: q.put(full, timeout=0.01) self.fail("Didn't appear to time-out with a full queue") except Queue.Full: pass # Test a blocking put self.do_blocking_test(q.put, (full,), q.get, ()) self.do_blocking_test(q.put, (full, True, 10), q.get, ()) # Empty it for i in range(QUEUE_SIZE): q.get() self.assertTrue(q.empty(), "Queue should be empty") try: q.get(block=0) self.fail("Didn't appear to block with an empty queue") except Queue.Empty: pass try: q.get(timeout=0.01) self.fail("Didn't appear to time-out with an empty queue") except Queue.Empty: pass # Test a blocking get self.do_blocking_test(q.get, (), q.put, ('empty',)) self.do_blocking_test(q.get, (True, 10), q.put, ('empty',)) def worker(self, q): while True: x = q.get() if x is None: q.task_done() return with self.cumlock: self.cum += x q.task_done() def queue_join_test(self, q): self.cum = 0 for i in (0,1): threading.Thread(target=self.worker, args=(q,)).start() for i in xrange(100): q.put(i) q.join() self.assertEqual(self.cum, sum(range(100)), "q.join() did not block until all tasks were done") for i in (0,1): q.put(None) # instruct the threads to close q.join() # verify that you can join twice def test_queue_task_done(self): # Test to make sure a queue task completed successfully. q = self.type2test() try: q.task_done() except ValueError: pass else: self.fail("Did not detect task count going negative") def test_queue_join(self): # Test that a queue join()s successfully, and before anything else # (done twice for insurance). q = self.type2test() self.queue_join_test(q) self.queue_join_test(q) try: q.task_done() except ValueError: pass else: self.fail("Did not detect task count going negative") def test_simple_queue(self): # Do it a couple of times on the same queue. # Done twice to make sure works with same instance reused. q = self.type2test(QUEUE_SIZE) self.simple_queue_test(q) self.simple_queue_test(q) class QueueTest(BaseQueueTest, unittest.TestCase): type2test = Queue.Queue class LifoQueueTest(BaseQueueTest, unittest.TestCase): type2test = Queue.LifoQueue class PriorityQueueTest(BaseQueueTest, unittest.TestCase): type2test = Queue.PriorityQueue # A Queue subclass that can provoke failure at a moment's notice :) class FailingQueueException(Exception): pass class FailingQueue(Queue.Queue): def __init__(self, *args): self.fail_next_put = False self.fail_next_get = False Queue.Queue.__init__(self, *args) def _put(self, item): if self.fail_next_put: self.fail_next_put = False raise FailingQueueException, "You Lose" return Queue.Queue._put(self, item) def _get(self): if self.fail_next_get: self.fail_next_get = False raise FailingQueueException, "You Lose" return Queue.Queue._get(self) class FailingQueueTest(BlockingTestMixin, unittest.TestCase): def failing_queue_test(self, q): if not q.empty(): raise RuntimeError, "Call this function with an empty queue" for i in range(QUEUE_SIZE-1): q.put(i) # Test a failing non-blocking put. q.fail_next_put = True try: q.put("oops", block=0) self.fail("The queue didn't fail when it should have") except FailingQueueException: pass q.fail_next_put = True try: q.put("oops", timeout=0.1) self.fail("The queue didn't fail when it should have") except FailingQueueException: pass q.put("last") self.assertTrue(q.full(), "Queue should be full") # Test a failing blocking put q.fail_next_put = True try: self.do_blocking_test(q.put, ("full",), q.get, ()) self.fail("The queue didn't fail when it should have") except FailingQueueException: pass # Check the Queue isn't damaged. # put failed, but get succeeded - re-add q.put("last") # Test a failing timeout put q.fail_next_put = True try: self.do_exceptional_blocking_test(q.put, ("full", True, 10), q.get, (), FailingQueueException) self.fail("The queue didn't fail when it should have") except FailingQueueException: pass # Check the Queue isn't damaged. # put failed, but get succeeded - re-add q.put("last") self.assertTrue(q.full(), "Queue should be full") q.get() self.assertTrue(not q.full(), "Queue should not be full") q.put("last") self.assertTrue(q.full(), "Queue should be full") # Test a blocking put self.do_blocking_test(q.put, ("full",), q.get, ()) # Empty it for i in range(QUEUE_SIZE): q.get() self.assertTrue(q.empty(), "Queue should be empty") q.put("first") q.fail_next_get = True try: q.get() self.fail("The queue didn't fail when it should have") except FailingQueueException: pass self.assertTrue(not q.empty(), "Queue should not be empty") q.fail_next_get = True try: q.get(timeout=0.1) self.fail("The queue didn't fail when it should have") except FailingQueueException: pass self.assertTrue(not q.empty(), "Queue should not be empty") q.get() self.assertTrue(q.empty(), "Queue should be empty") q.fail_next_get = True try: self.do_exceptional_blocking_test(q.get, (), q.put, ('empty',), FailingQueueException) self.fail("The queue didn't fail when it should have") except FailingQueueException: pass # put succeeded, but get failed. self.assertTrue(not q.empty(), "Queue should not be empty") q.get() self.assertTrue(q.empty(), "Queue should be empty") def test_failing_queue(self): # Test to make sure a queue is functioning correctly. # Done twice to the same instance. q = FailingQueue(QUEUE_SIZE) self.failing_queue_test(q) self.failing_queue_test(q) def test_main(): test_support.run_unittest(QueueTest, LifoQueueTest, PriorityQueueTest, FailingQueueTest) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_quopri.py ================================================ from test import test_support import unittest import sys, cStringIO #, subprocess import quopri ENCSAMPLE = """\ Here's a bunch of special=20 =A1=A2=A3=A4=A5=A6=A7=A8=A9 =AA=AB=AC=AD=AE=AF=B0=B1=B2=B3 =B4=B5=B6=B7=B8=B9=BA=BB=BC=BD=BE =BF=C0=C1=C2=C3=C4=C5=C6 =C7=C8=C9=CA=CB=CC=CD=CE=CF =D0=D1=D2=D3=D4=D5=D6=D7 =D8=D9=DA=DB=DC=DD=DE=DF =E0=E1=E2=E3=E4=E5=E6=E7 =E8=E9=EA=EB=EC=ED=EE=EF =F0=F1=F2=F3=F4=F5=F6=F7 =F8=F9=FA=FB=FC=FD=FE=FF characters... have fun! """ # First line ends with a space DECSAMPLE = "Here's a bunch of special \n" + \ """\ \xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9 \xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3 \xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe \xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6 \xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf \xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7 \xd8\xd9\xda\xdb\xdc\xdd\xde\xdf \xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7 \xe8\xe9\xea\xeb\xec\xed\xee\xef \xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7 \xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff characters... have fun! """ def withpythonimplementation(testfunc): def newtest(self): # Test default implementation testfunc(self) # Test Python implementation if quopri.b2a_qp is not None or quopri.a2b_qp is not None: oldencode = quopri.b2a_qp olddecode = quopri.a2b_qp try: quopri.b2a_qp = None quopri.a2b_qp = None testfunc(self) finally: quopri.b2a_qp = oldencode quopri.a2b_qp = olddecode #newtest.__name__ = testfunc.__name__ return newtest class QuopriTestCase(unittest.TestCase): # Each entry is a tuple of (plaintext, encoded string). These strings are # used in the "quotetabs=0" tests. STRINGS = ( # Some normal strings ('hello', 'hello'), ('''hello there world''', '''hello there world'''), ('''hello there world ''', '''hello there world '''), ('\201\202\203', '=81=82=83'), # Add some trailing MUST QUOTE strings ('hello ', 'hello=20'), ('hello\t', 'hello=09'), # Some long lines. First, a single line of 108 characters ('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\xd8\xd9\xda\xdb\xdc\xdd\xde\xdfxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', '''xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=D8=D9=DA=DB=DC=DD=DE=DFx= xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'''), # A line of exactly 76 characters, no soft line break should be needed ('yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy', 'yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy'), # A line of 77 characters, forcing a soft line break at position 75, # and a second line of exactly 2 characters (because the soft line # break `=' sign counts against the line length limit). ('zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', '''zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz= zz'''), # A line of 151 characters, forcing a soft line break at position 75, # with a second line of exactly 76 characters and no trailing = ('zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', '''zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz= zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'''), # A string containing a hard line break, but which the first line is # 151 characters and the second line is exactly 76 characters. This # should leave us with three lines, the first which has a soft line # break, and which the second and third do not. ('''yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz''', '''yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy= yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'''), # Now some really complex stuff ;) (DECSAMPLE, ENCSAMPLE), ) # These are used in the "quotetabs=1" tests. ESTRINGS = ( ('hello world', 'hello=20world'), ('hello\tworld', 'hello=09world'), ) # These are used in the "header=1" tests. HSTRINGS = ( ('hello world', 'hello_world'), ('hello_world', 'hello=5Fworld'), ) @withpythonimplementation def test_encodestring(self): for p, e in self.STRINGS: self.assertTrue(quopri.encodestring(p) == e) @withpythonimplementation def test_decodestring(self): for p, e in self.STRINGS: self.assertTrue(quopri.decodestring(e) == p) @withpythonimplementation def test_idempotent_string(self): for p, e in self.STRINGS: self.assertTrue(quopri.decodestring(quopri.encodestring(e)) == e) @withpythonimplementation def test_encode(self): for p, e in self.STRINGS: infp = cStringIO.StringIO(p) outfp = cStringIO.StringIO() quopri.encode(infp, outfp, quotetabs=False) self.assertTrue(outfp.getvalue() == e) @withpythonimplementation def test_decode(self): for p, e in self.STRINGS: infp = cStringIO.StringIO(e) outfp = cStringIO.StringIO() quopri.decode(infp, outfp) self.assertTrue(outfp.getvalue() == p) @withpythonimplementation def test_embedded_ws(self): for p, e in self.ESTRINGS: self.assertTrue(quopri.encodestring(p, quotetabs=True) == e) self.assertTrue(quopri.decodestring(e) == p) @withpythonimplementation def test_encode_header(self): for p, e in self.HSTRINGS: self.assertTrue(quopri.encodestring(p, header=True) == e) @withpythonimplementation def test_decode_header(self): for p, e in self.HSTRINGS: self.assertTrue(quopri.decodestring(e, header=True) == p) @unittest.expectedFailure def test_scriptencode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen([sys.executable, "-mquopri"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) self.addCleanup(process.stdout.close) cout, cerr = process.communicate(p) # On Windows, Python will output the result to stdout using # CRLF, as the mode of stdout is text mode. To compare this # with the expected result, we need to do a line-by-line comparison. self.assertEqual(cout.splitlines(), e.splitlines()) @unittest.expectedFailure def test_scriptdecode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen([sys.executable, "-mquopri", "-d"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) self.addCleanup(process.stdout.close) cout, cerr = process.communicate(e) self.assertEqual(cout.splitlines(), p.splitlines()) def test_main(): test_support.run_unittest(QuopriTestCase) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_rfc822.py ================================================ import unittest from test import test_support #rfc822 = test_support.import_module("rfc822", deprecated=True) import rfc822 try: from cStringIO import StringIO except ImportError: from StringIO import StringIO class MessageTestCase(unittest.TestCase): def create_message(self, msg): return rfc822.Message(StringIO(msg)) def test_get(self): msg = self.create_message( 'To: "last, first" \n\ntest\n') self.assertTrue(msg.get("to") == '"last, first" ') self.assertTrue(msg.get("TO") == '"last, first" ') self.assertTrue(msg.get("No-Such-Header") is None) self.assertTrue(msg.get("No-Such-Header", "No-Such-Value") == "No-Such-Value") def test_setdefault(self): msg = self.create_message( 'To: "last, first" \n\ntest\n') self.assertTrue(not msg.has_key("New-Header")) self.assertTrue(msg.setdefault("New-Header", "New-Value") == "New-Value") self.assertTrue(msg.setdefault("New-Header", "Different-Value") == "New-Value") self.assertTrue(msg["new-header"] == "New-Value") self.assertTrue(msg.setdefault("Another-Header") == "") self.assertTrue(msg["another-header"] == "") def check(self, msg, results): """Check addresses and the date.""" m = self.create_message(msg) i = 0 for n, a in m.getaddrlist('to') + m.getaddrlist('cc'): try: mn, ma = results[i][0], results[i][1] except IndexError: print 'extra parsed address:', repr(n), repr(a) continue i = i + 1 self.assertEqual(mn, n, "Un-expected name: %r != %r" % (mn, n)) self.assertEqual(ma, a, "Un-expected address: %r != %r" % (ma, a)) if mn == n and ma == a: pass else: print 'not found:', repr(n), repr(a) out = m.getdate('date') if out: self.assertEqual(out, (1999, 1, 13, 23, 57, 35, 0, 1, 0), "date conversion failed") # Note: all test cases must have the same date (in various formats), # or no date! def test_basic(self): self.check( 'Date: Wed, 13 Jan 1999 23:57:35 -0500\n' 'From: Guido van Rossum \n' 'To: "Guido van\n' '\t : Rossum" \n' 'Subject: test2\n' '\n' 'test2\n', [('Guido van\n\t : Rossum', 'guido@python.org')]) self.check( 'From: Barry \n' 'Date: 13-Jan-1999 23:57:35 EST\n' '\n' 'test', [('Guido: the Barbarian', 'guido@python.org'), ('Guido: the Madman', 'guido@python.org') ]) self.check( 'To: "The monster with\n' ' the very long name: Guido" \n' 'Date: Wed, 13 Jan 1999 23:57:35 -0500\n' '\n' 'test', [('The monster with\n the very long name: Guido', 'guido@python.org')]) self.check( 'To: "Amit J. Patel" \n' 'CC: Mike Fletcher ,\n' ' "\'string-sig@python.org\'" \n' 'Cc: fooz@bat.com, bart@toof.com\n' 'Cc: goit@lip.com\n' 'Date: Wed, 13 Jan 1999 23:57:35 -0500\n' '\n' 'test', [('Amit J. Patel', 'amitp@Theory.Stanford.EDU'), ('Mike Fletcher', 'mfletch@vrtelecom.com'), ("'string-sig@python.org'", 'string-sig@python.org'), ('', 'fooz@bat.com'), ('', 'bart@toof.com'), ('', 'goit@lip.com'), ]) self.check( 'To: Some One \n' 'From: Anudder Persin \n' 'Date:\n' '\n' 'test', [('Some One', 'someone@dom.ain')]) self.check( 'To: person@dom.ain (User J. Person)\n\n', [('User J. Person', 'person@dom.ain')]) def test_doublecomment(self): # The RFC allows comments within comments in an email addr self.check( 'To: person@dom.ain ((User J. Person)), John Doe \n\n', [('User J. Person', 'person@dom.ain'), ('John Doe', 'foo@bar.com')]) def test_twisted(self): # This one is just twisted. I don't know what the proper # result should be, but it shouldn't be to infloop, which is # what used to happen! self.check( 'To: <[smtp:dd47@mail.xxx.edu]_at_hmhq@hdq-mdm1-imgout.companay.com>\n' 'Date: Wed, 13 Jan 1999 23:57:35 -0500\n' '\n' 'test', [('', ''), ('', 'dd47@mail.xxx.edu'), ('', '_at_hmhq@hdq-mdm1-imgout.companay.com'), ]) def test_commas_in_full_name(self): # This exercises the old commas-in-a-full-name bug, which # should be doing the right thing in recent versions of the # module. self.check( 'To: "last, first" \n' '\n' 'test', [('last, first', 'userid@foo.net')]) def test_quoted_name(self): self.check( 'To: (Comment stuff) "Quoted name"@somewhere.com\n' '\n' 'test', [('Comment stuff', '"Quoted name"@somewhere.com')]) def test_bogus_to_header(self): self.check( 'To: :\n' 'Cc: goit@lip.com\n' 'Date: Wed, 13 Jan 1999 23:57:35 -0500\n' '\n' 'test', [('', 'goit@lip.com')]) def test_addr_ipquad(self): self.check( 'To: guido@[132.151.1.21]\n' '\n' 'foo', [('', 'guido@[132.151.1.21]')]) def test_iter(self): m = rfc822.Message(StringIO( 'Date: Wed, 13 Jan 1999 23:57:35 -0500\n' 'From: Guido van Rossum \n' 'To: "Guido van\n' '\t : Rossum" \n' 'Subject: test2\n' '\n' 'test2\n' )) self.assertEqual(sorted(m), ['date', 'from', 'subject', 'to']) def test_rfc2822_phrases(self): # RFC 2822 (the update to RFC 822) specifies that dots in phrases are # obsolete syntax, which conforming programs MUST recognize but NEVER # generate (see $4.1 Miscellaneous obsolete tokens). This is a # departure from RFC 822 which did not allow dots in non-quoted # phrases. self.check('To: User J. Person \n\n', [('User J. Person', 'person@dom.ain')]) # This takes too long to add to the test suite ## def test_an_excrutiatingly_long_address_field(self): ## OBSCENELY_LONG_HEADER_MULTIPLIER = 10000 ## oneaddr = ('Person' * 10) + '@' + ('.'.join(['dom']*10)) + '.com' ## addr = ', '.join([oneaddr] * OBSCENELY_LONG_HEADER_MULTIPLIER) ## lst = rfc822.AddrlistClass(addr).getaddrlist() ## self.assertEqual(len(lst), OBSCENELY_LONG_HEADER_MULTIPLIER) def test_2getaddrlist(self): eq = self.assertEqual msg = self.create_message("""\ To: aperson@dom.ain Cc: bperson@dom.ain Cc: cperson@dom.ain Cc: dperson@dom.ain A test message. """) ccs = [('', a) for a in ['bperson@dom.ain', 'cperson@dom.ain', 'dperson@dom.ain']] addrs = msg.getaddrlist('cc') addrs.sort() eq(addrs, ccs) # Try again, this one used to fail addrs = msg.getaddrlist('cc') addrs.sort() eq(addrs, ccs) def test_parseaddr(self): eq = self.assertEqual eq(rfc822.parseaddr('<>'), ('', '')) eq(rfc822.parseaddr('aperson@dom.ain'), ('', 'aperson@dom.ain')) eq(rfc822.parseaddr('bperson@dom.ain (Bea A. Person)'), ('Bea A. Person', 'bperson@dom.ain')) eq(rfc822.parseaddr('Cynthia Person '), ('Cynthia Person', 'cperson@dom.ain')) def test_quote_unquote(self): eq = self.assertEqual eq(rfc822.quote('foo\\wacky"name'), 'foo\\\\wacky\\"name') eq(rfc822.unquote('"foo\\\\wacky\\"name"'), 'foo\\wacky"name') def test_invalid_headers(self): eq = self.assertEqual msg = self.create_message("First: val\n: otherval\nSecond: val2\n") eq(msg.getheader('First'), 'val') eq(msg.getheader('Second'), 'val2') def test_main(): test_support.run_unittest(MessageTestCase) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_sched.py ================================================ #import Queue as queue import sched import time import unittest import test.test_support try: import threading except ImportError: threading = None TIMEOUT = 10 class Timer(object): def __init__(self): self._cond = threading.Condition() self._time = 0 self._stop = 0 def time(self): with self._cond: return self._time # increase the time but not beyond the established limit def sleep(self, t): assert t >= 0 with self._cond: t += self._time while self._stop < t: self._time = self._stop self._cond.wait() self._time = t # advance time limit for user code def advance(self, t): assert t >= 0 with self._cond: self._stop += t self._cond.notify_all() class TestCase(unittest.TestCase): def test_enter(self): l = [] fun = lambda x: l.append(x) scheduler = sched.scheduler(time.time, time.sleep) for x in [0.5, 0.4, 0.3, 0.2, 0.1]: z = scheduler.enter(x, 1, fun, (x,)) scheduler.run() self.assertEqual(l, [0.1, 0.2, 0.3, 0.4, 0.5]) def test_enterabs(self): l = [] fun = lambda x: l.append(x) scheduler = sched.scheduler(time.time, time.sleep) for x in [0.05, 0.04, 0.03, 0.02, 0.01]: z = scheduler.enterabs(x, 1, fun, (x,)) scheduler.run() self.assertEqual(l, [0.01, 0.02, 0.03, 0.04, 0.05]) #@unittest.skipUnless(threading, 'Threading required for this test.') @unittest.skip('grumpy') def test_enter_concurrent(self): q = queue.Queue() fun = q.put timer = Timer() scheduler = sched.scheduler(timer.time, timer.sleep) scheduler.enter(1, 1, fun, (1,)) scheduler.enter(3, 1, fun, (3,)) t = threading.Thread(target=scheduler.run) t.start() timer.advance(1) self.assertEqual(q.get(timeout=TIMEOUT), 1) self.assertTrue(q.empty()) for x in [4, 5, 2]: z = scheduler.enter(x - 1, 1, fun, (x,)) timer.advance(2) self.assertEqual(q.get(timeout=TIMEOUT), 2) self.assertEqual(q.get(timeout=TIMEOUT), 3) self.assertTrue(q.empty()) timer.advance(1) self.assertEqual(q.get(timeout=TIMEOUT), 4) self.assertTrue(q.empty()) timer.advance(1) self.assertEqual(q.get(timeout=TIMEOUT), 5) self.assertTrue(q.empty()) timer.advance(1000) t.join(timeout=TIMEOUT) self.assertFalse(t.is_alive()) self.assertTrue(q.empty()) self.assertEqual(timer.time(), 5) def test_priority(self): l = [] fun = lambda x: l.append(x) scheduler = sched.scheduler(time.time, time.sleep) for priority in [1, 2, 3, 4, 5]: z = scheduler.enterabs(0.01, priority, fun, (priority,)) scheduler.run() self.assertEqual(l, [1, 2, 3, 4, 5]) @unittest.skip('grumpy') def test_cancel(self): l = [] fun = lambda x: l.append(x) scheduler = sched.scheduler(time.time, time.sleep) now = time.time() event1 = scheduler.enterabs(now + 0.01, 1, fun, (0.01,)) event2 = scheduler.enterabs(now + 0.02, 1, fun, (0.02,)) event3 = scheduler.enterabs(now + 0.03, 1, fun, (0.03,)) event4 = scheduler.enterabs(now + 0.04, 1, fun, (0.04,)) event5 = scheduler.enterabs(now + 0.05, 1, fun, (0.05,)) scheduler.cancel(event1) scheduler.cancel(event5) scheduler.run() self.assertEqual(l, [0.02, 0.03, 0.04]) #@unittest.skipUnless(threading, 'Threading required for this test.') @unittest.skip('grumpy') def test_cancel_concurrent(self): q = queue.Queue() fun = q.put timer = Timer() scheduler = sched.scheduler(timer.time, timer.sleep) now = timer.time() event1 = scheduler.enterabs(now + 1, 1, fun, (1,)) event2 = scheduler.enterabs(now + 2, 1, fun, (2,)) event4 = scheduler.enterabs(now + 4, 1, fun, (4,)) event5 = scheduler.enterabs(now + 5, 1, fun, (5,)) event3 = scheduler.enterabs(now + 3, 1, fun, (3,)) t = threading.Thread(target=scheduler.run) t.start() timer.advance(1) self.assertEqual(q.get(timeout=TIMEOUT), 1) self.assertTrue(q.empty()) scheduler.cancel(event2) scheduler.cancel(event5) timer.advance(1) self.assertTrue(q.empty()) timer.advance(1) self.assertEqual(q.get(timeout=TIMEOUT), 3) self.assertTrue(q.empty()) timer.advance(1) self.assertEqual(q.get(timeout=TIMEOUT), 4) self.assertTrue(q.empty()) timer.advance(1000) t.join(timeout=TIMEOUT) self.assertFalse(t.is_alive()) self.assertTrue(q.empty()) self.assertEqual(timer.time(), 4) def test_empty(self): l = [] fun = lambda x: l.append(x) scheduler = sched.scheduler(time.time, time.sleep) self.assertTrue(scheduler.empty()) for x in [0.05, 0.04, 0.03, 0.02, 0.01]: z = scheduler.enterabs(x, 1, fun, (x,)) self.assertFalse(scheduler.empty()) scheduler.run() self.assertTrue(scheduler.empty()) def test_main(): test.test_support.run_unittest(TestCase) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_select.py ================================================ from test import test_support import unittest import select_ as select import os import sys @unittest.skipIf(sys.platform[:3] in ('win', 'os2', 'riscos'), "can't easily test on this system") class SelectTestCase(unittest.TestCase): class Nope(object): pass class Almost(object): def fileno(self): return 'fileno' def test_error_conditions(self): self.assertRaises(TypeError, select.select, 1, 2, 3) self.assertRaises(TypeError, select.select, [self.Nope()], [], []) self.assertRaises(TypeError, select.select, [self.Almost()], [], []) self.assertRaises(ValueError, select.select, [], [], [], "not a number") def test_returned_list_identity(self): # See issue #8329 r, w, x = select.select([], [], [], 1) self.assertIsNot(r, w) self.assertIsNot(r, x) self.assertIsNot(w, x) def test_select(self): cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' p = os.popen(cmd, 'r') for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: if test_support.verbose: print 'timeout =', tout rfd, wfd, xfd = select.select([p], [], [], tout) if (rfd, wfd, xfd) == ([], [], []): continue if (rfd, wfd, xfd) == ([p], [], []): line = p.readline() if test_support.verbose: print repr(line) if not line: if test_support.verbose: print 'EOF' break continue self.fail('Unexpected return values from select():', rfd, wfd, xfd) p.close() # Issue 16230: Crash on select resized list def test_select_mutated(self): a = [] class F(object): def fileno(self): del a[-1] return sys.stdout.fileno() a[:] = [F()] * 10 self.assertEqual(select.select([], a, []), ([], a[:5], [])) def test_main(): test_support.run_unittest(SelectTestCase) test_support.reap_children() if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_slice.py ================================================ # tests for slice objects; in particular the indices method. import unittest import weakref # from cPickle import loads, dumps from test import test_support import sys class SliceTest(unittest.TestCase): def test_constructor(self): self.assertRaises(TypeError, slice) self.assertRaises(TypeError, slice, 1, 2, 3, 4) def test_repr(self): self.assertEqual(repr(slice(1, 2, 3)), "slice(1, 2, 3)") def test_hash(self): # Verify clearing of SF bug #800796 self.assertRaises(TypeError, hash, slice(5)) with self.assertRaises(TypeError): slice(5).__hash__() @unittest.expectedFailure def test_cmp(self): s1 = slice(1, 2, 3) s2 = slice(1, 2, 3) s3 = slice(1, 2, 4) self.assertEqual(s1, s2) self.assertNotEqual(s1, s3) class Exc(Exception): pass class BadCmp(object): def __eq__(self, other): raise Exc __hash__ = None # Silence Py3k warning s1 = slice(BadCmp()) s2 = slice(BadCmp()) self.assertRaises(Exc, cmp, s1, s2) self.assertEqual(s1, s1) s1 = slice(1, BadCmp()) s2 = slice(1, BadCmp()) self.assertEqual(s1, s1) self.assertRaises(Exc, cmp, s1, s2) s1 = slice(1, 2, BadCmp()) s2 = slice(1, 2, BadCmp()) self.assertEqual(s1, s1) self.assertRaises(Exc, cmp, s1, s2) def test_members(self): s = slice(1) self.assertEqual(s.start, None) self.assertEqual(s.stop, 1) self.assertEqual(s.step, None) s = slice(1, 2) self.assertEqual(s.start, 1) self.assertEqual(s.stop, 2) self.assertEqual(s.step, None) s = slice(1, 2, 3) self.assertEqual(s.start, 1) self.assertEqual(s.stop, 2) self.assertEqual(s.step, 3) class AnyClass(object): pass obj = AnyClass() s = slice(obj) self.assertTrue(s.stop is obj) @unittest.expectedFailure def test_indices(self): self.assertEqual(slice(None ).indices(10), (0, 10, 1)) self.assertEqual(slice(None, None, 2).indices(10), (0, 10, 2)) self.assertEqual(slice(1, None, 2).indices(10), (1, 10, 2)) self.assertEqual(slice(None, None, -1).indices(10), (9, -1, -1)) self.assertEqual(slice(None, None, -2).indices(10), (9, -1, -2)) self.assertEqual(slice(3, None, -2).indices(10), (3, -1, -2)) # issue 3004 tests self.assertEqual(slice(None, -9).indices(10), (0, 1, 1)) self.assertEqual(slice(None, -10).indices(10), (0, 0, 1)) self.assertEqual(slice(None, -11).indices(10), (0, 0, 1)) self.assertEqual(slice(None, -10, -1).indices(10), (9, 0, -1)) self.assertEqual(slice(None, -11, -1).indices(10), (9, -1, -1)) self.assertEqual(slice(None, -12, -1).indices(10), (9, -1, -1)) self.assertEqual(slice(None, 9).indices(10), (0, 9, 1)) self.assertEqual(slice(None, 10).indices(10), (0, 10, 1)) self.assertEqual(slice(None, 11).indices(10), (0, 10, 1)) self.assertEqual(slice(None, 8, -1).indices(10), (9, 8, -1)) self.assertEqual(slice(None, 9, -1).indices(10), (9, 9, -1)) self.assertEqual(slice(None, 10, -1).indices(10), (9, 9, -1)) self.assertEqual( slice(-100, 100 ).indices(10), slice(None).indices(10) ) self.assertEqual( slice(100, -100, -1).indices(10), slice(None, None, -1).indices(10) ) self.assertEqual(slice(-100L, 100L, 2L).indices(10), (0, 10, 2)) self.assertEqual(range(10)[::sys.maxint - 1], [0]) self.assertRaises(OverflowError, slice(None).indices, 1L<<100) @unittest.expectedFailure def test_setslice_without_getslice(self): tmp = [] class X(object): def __setslice__(self, i, j, k): tmp.append((i, j, k)) x = X() with test_support.check_py3k_warnings(): x[1:2] = 42 self.assertEqual(tmp, [(1, 2, 42)]) # def test_pickle(self): # s = slice(10, 20, 3) # for protocol in (0,1,2): # t = loads(dumps(s, protocol)) # self.assertEqual(s, t) # self.assertEqual(s.indices(15), t.indices(15)) # self.assertNotEqual(id(s), id(t)) @unittest.expectedFailure def test_cycle(self): class myobj(object): pass o = myobj() o.s = slice(o) w = weakref.ref(o) o = None test_support.gc_collect() self.assertIsNone(w()) def test_main(): test_support.run_unittest(SliceTest) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_stat.py ================================================ import unittest import os from test.test_support import TESTFN, run_unittest import stat class TestFilemode(unittest.TestCase): file_flags = {'SF_APPEND', 'SF_ARCHIVED', 'SF_IMMUTABLE', 'SF_NOUNLINK', 'SF_SNAPSHOT', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE'} formats = {'S_IFBLK', 'S_IFCHR', 'S_IFDIR', 'S_IFIFO', 'S_IFLNK', 'S_IFREG', 'S_IFSOCK'} format_funcs = {'S_ISBLK', 'S_ISCHR', 'S_ISDIR', 'S_ISFIFO', 'S_ISLNK', 'S_ISREG', 'S_ISSOCK'} stat_struct = { 'ST_MODE': 0, 'ST_INO': 1, 'ST_DEV': 2, 'ST_NLINK': 3, 'ST_UID': 4, 'ST_GID': 5, 'ST_SIZE': 6, 'ST_ATIME': 7, 'ST_MTIME': 8, 'ST_CTIME': 9} # permission bit value are defined by POSIX permission_bits = { 'S_ISUID': 0o4000, 'S_ISGID': 0o2000, 'S_ENFMT': 0o2000, 'S_ISVTX': 0o1000, 'S_IRWXU': 0o700, 'S_IRUSR': 0o400, 'S_IREAD': 0o400, 'S_IWUSR': 0o200, 'S_IWRITE': 0o200, 'S_IXUSR': 0o100, 'S_IEXEC': 0o100, 'S_IRWXG': 0o070, 'S_IRGRP': 0o040, 'S_IWGRP': 0o020, 'S_IXGRP': 0o010, 'S_IRWXO': 0o007, 'S_IROTH': 0o004, 'S_IWOTH': 0o002, 'S_IXOTH': 0o001} def setUp(self): try: os.remove(TESTFN) except OSError: try: os.rmdir(TESTFN) except OSError: pass tearDown = setUp def get_mode(self, fname=TESTFN): #, lstat=True): # if lstat: # st_mode = os.lstat(fname).st_mode # else: st_mode = os.stat(fname).st_mode return st_mode def assertS_IS(self, name, mode): # test format, lstrip is for S_IFIFO # fmt = getattr(stat, "S_IF" + name.lstrip("F")) # self.assertEqual(stat.S_IFMT(mode), fmt) # test that just one function returns true testname = "S_IS" + name for funcname in self.format_funcs: func = getattr(stat, funcname, None) if func is None: if funcname == testname: raise ValueError(funcname) continue if funcname == testname: self.assertTrue(func(mode)) else: self.assertFalse(func(mode)) @unittest.skip('grumpy') def test_mode(self): with open(TESTFN, 'w'): pass if os.name == 'posix': os.chmod(TESTFN, 0o700) st_mode = self.get_mode() self.assertS_IS("REG", st_mode) self.assertEqual(stat.S_IMODE(st_mode), stat.S_IRWXU) os.chmod(TESTFN, 0o070) st_mode = self.get_mode() self.assertS_IS("REG", st_mode) self.assertEqual(stat.S_IMODE(st_mode), stat.S_IRWXG) os.chmod(TESTFN, 0o007) st_mode = self.get_mode() self.assertS_IS("REG", st_mode) self.assertEqual(stat.S_IMODE(st_mode), stat.S_IRWXO) os.chmod(TESTFN, 0o444) st_mode = self.get_mode() self.assertS_IS("REG", st_mode) self.assertEqual(stat.S_IMODE(st_mode), 0o444) else: os.chmod(TESTFN, 0o700) st_mode = self.get_mode() self.assertS_IS("REG", st_mode) self.assertEqual(stat.S_IFMT(st_mode), stat.S_IFREG) def test_directory(self): os.mkdir(TESTFN) os.chmod(TESTFN, 0o700) st_mode = self.get_mode() self.assertS_IS("DIR", st_mode) @unittest.skip('grumpy') @unittest.skipUnless(hasattr(os, 'symlink'), 'os.symlink not available') def test_link(self): try: os.symlink(os.getcwd(), TESTFN) except (OSError, NotImplementedError) as err: raise unittest.SkipTest(str(err)) else: st_mode = self.get_mode() self.assertS_IS("LNK", st_mode) @unittest.skip('grumpy') @unittest.skipUnless(hasattr(os, 'mkfifo'), 'os.mkfifo not available') def test_fifo(self): os.mkfifo(TESTFN, 0o700) st_mode = self.get_mode() self.assertS_IS("FIFO", st_mode) @unittest.skip('grumpy') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_devices(self): if os.path.exists(os.devnull): st_mode = self.get_mode(os.devnull, lstat=False) self.assertS_IS("CHR", st_mode) # Linux block devices, BSD has no block devices anymore for blockdev in ("/dev/sda", "/dev/hda"): if os.path.exists(blockdev): st_mode = self.get_mode(blockdev, lstat=False) self.assertS_IS("BLK", st_mode) break @unittest.skip('grumpy') def test_module_attributes(self): for key, value in self.stat_struct.items(): modvalue = getattr(stat, key) self.assertEqual(value, modvalue, key) for key, value in self.permission_bits.items(): modvalue = getattr(stat, key) self.assertEqual(value, modvalue, key) for key in self.file_flags: modvalue = getattr(stat, key) self.assertIsInstance(modvalue, int) for key in self.formats: modvalue = getattr(stat, key) self.assertIsInstance(modvalue, int) for key in self.format_funcs: func = getattr(stat, key) self.assertTrue(callable(func)) self.assertEqual(func(0), 0) def test_main(): run_unittest(TestFilemode) if __name__ == '__main__': test_main() ================================================ FILE: third_party/stdlib/test/test_string.py ================================================ import unittest import string # from string import Template Template = string.Template from test import test_support, string_tests # from UserList import UserList import UserList as _UserList UserList = _UserList.UserList class StringTest( string_tests.CommonTest, string_tests.MixinStrStringUserStringTest ): type2test = str def checkequal(self, result, object, methodname, *args): realresult = getattr(string, methodname)(object, *args) self.assertEqual( result, realresult ) def checkraises(self, exc, obj, methodname, *args): with self.assertRaises(exc) as cm: getattr(string, methodname)(obj, *args) self.assertNotEqual(cm.exception.args[0], '') def checkcall(self, object, methodname, *args): getattr(string, methodname)(object, *args) @unittest.expectedFailure def test_join(self): # These are the same checks as in string_test.ObjectTest.test_join # but the argument order ist different self.checkequal('a b c d', ['a', 'b', 'c', 'd'], 'join', ' ') self.checkequal('abcd', ('a', 'b', 'c', 'd'), 'join', '') self.checkequal('w x y z', string_tests.Sequence(), 'join', ' ') self.checkequal('abc', ('abc',), 'join', 'a') self.checkequal('z', UserList(['z']), 'join', 'a') if test_support.have_unicode: self.checkequal(unicode('a.b.c'), ['a', 'b', 'c'], 'join', unicode('.')) self.checkequal(unicode('a.b.c'), [unicode('a'), 'b', 'c'], 'join', '.') self.checkequal(unicode('a.b.c'), ['a', unicode('b'), 'c'], 'join', '.') self.checkequal(unicode('a.b.c'), ['a', 'b', unicode('c')], 'join', '.') self.checkraises(TypeError, ['a', unicode('b'), 3], 'join', '.') for i in [5, 25, 125]: self.checkequal( ((('a' * i) + '-') * i)[:-1], ['a' * i] * i, 'join', '-') self.checkequal( ((('a' * i) + '-') * i)[:-1], ('a' * i,) * i, 'join', '-') self.checkraises(TypeError, string_tests.BadSeq1(), 'join', ' ') self.checkequal('a b c', string_tests.BadSeq2(), 'join', ' ') try: def f(): yield 4 + "" self.fixtype(' ').join(f()) except TypeError, e: if '+' not in str(e): self.fail('join() ate exception message') else: self.fail('exception not raised') class ModuleTest(unittest.TestCase): def test_attrs(self): string.whitespace string.lowercase string.uppercase string.letters string.digits string.hexdigits string.octdigits string.punctuation string.printable def test_atoi(self): self.assertEqual(string.atoi(" 1 "), 1) self.assertRaises(ValueError, string.atoi, " 1x") self.assertRaises(ValueError, string.atoi, " x1 ") def test_atol(self): self.assertEqual(string.atol(" 1 "), 1L) self.assertRaises(ValueError, string.atol, " 1x ") self.assertRaises(ValueError, string.atol, " x1 ") @unittest.expectedFailure def test_atof(self): self.assertAlmostEqual(string.atof(" 1 "), 1.0) self.assertRaises(ValueError, string.atof, " 1x ") self.assertRaises(ValueError, string.atof, " x1 ") def test_maketrans(self): transtable = '\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037 !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`xyzdefghijklmnopqrstuvwxyz{|}~\177\200\201\202\203\204\205\206\207\210\211\212\213\214\215\216\217\220\221\222\223\224\225\226\227\230\231\232\233\234\235\236\237\240\241\242\243\244\245\246\247\250\251\252\253\254\255\256\257\260\261\262\263\264\265\266\267\270\271\272\273\274\275\276\277\300\301\302\303\304\305\306\307\310\311\312\313\314\315\316\317\320\321\322\323\324\325\326\327\330\331\332\333\334\335\336\337\340\341\342\343\344\345\346\347\350\351\352\353\354\355\356\357\360\361\362\363\364\365\366\367\370\371\372\373\374\375\376\377' self.assertEqual(string.maketrans('abc', 'xyz'), transtable) self.assertRaises(ValueError, string.maketrans, 'abc', 'xyzq') @unittest.expectedFailure def test_capwords(self): self.assertEqual(string.capwords('abc def ghi'), 'Abc Def Ghi') self.assertEqual(string.capwords('abc\tdef\nghi'), 'Abc Def Ghi') self.assertEqual(string.capwords('abc\t def \nghi'), 'Abc Def Ghi') self.assertEqual(string.capwords('ABC DEF GHI'), 'Abc Def Ghi') self.assertEqual(string.capwords('ABC-DEF-GHI', '-'), 'Abc-Def-Ghi') self.assertEqual(string.capwords('ABC-def DEF-ghi GHI'), 'Abc-def Def-ghi Ghi') self.assertEqual(string.capwords(' aBc DeF '), 'Abc Def') self.assertEqual(string.capwords('\taBc\tDeF\t'), 'Abc Def') self.assertEqual(string.capwords('\taBc\tDeF\t', '\t'), '\tAbc\tDef\t') @unittest.expectedFailure def test_formatter(self): fmt = string.Formatter() self.assertEqual(fmt.format("foo"), "foo") self.assertEqual(fmt.format("foo{0}", "bar"), "foobar") self.assertEqual(fmt.format("foo{1}{0}-{1}", "bar", 6), "foo6bar-6") self.assertEqual(fmt.format("-{arg!r}-", arg='test'), "-'test'-") # override get_value ############################################ class NamespaceFormatter(string.Formatter): def __init__(self, namespace={}): string.Formatter.__init__(self) self.namespace = namespace def get_value(self, key, args, kwds): if isinstance(key, str): try: # Check explicitly passed arguments first return kwds[key] except KeyError: return self.namespace[key] else: string.Formatter.get_value(key, args, kwds) fmt = NamespaceFormatter({'greeting':'hello'}) self.assertEqual(fmt.format("{greeting}, world!"), 'hello, world!') # override format_field ######################################### class CallFormatter(string.Formatter): def format_field(self, value, format_spec): return format(value(), format_spec) fmt = CallFormatter() self.assertEqual(fmt.format('*{0}*', lambda : 'result'), '*result*') # override convert_field ######################################## class XFormatter(string.Formatter): def convert_field(self, value, conversion): if conversion == 'x': return None return super(XFormatter, self).convert_field(value, conversion) fmt = XFormatter() self.assertEqual(fmt.format("{0!r}:{0!x}", 'foo', 'foo'), "'foo':None") # override parse ################################################ class BarFormatter(string.Formatter): # returns an iterable that contains tuples of the form: # (literal_text, field_name, format_spec, conversion) def parse(self, format_string): for field in format_string.split('|'): if field[0] == '+': # it's markup field_name, _, format_spec = field[1:].partition(':') yield '', field_name, format_spec, None else: yield field, None, None, None fmt = BarFormatter() self.assertEqual(fmt.format('*|+0:^10s|*', 'foo'), '* foo *') # test all parameters used class CheckAllUsedFormatter(string.Formatter): def check_unused_args(self, used_args, args, kwargs): # Track which arguments actually got used unused_args = set(kwargs.keys()) unused_args.update(range(0, len(args))) for arg in used_args: unused_args.remove(arg) if unused_args: raise ValueError("unused arguments") fmt = CheckAllUsedFormatter() self.assertEqual(fmt.format("{0}", 10), "10") self.assertEqual(fmt.format("{0}{i}", 10, i=100), "10100") self.assertEqual(fmt.format("{0}{i}{1}", 10, 20, i=100), "1010020") self.assertRaises(ValueError, fmt.format, "{0}{i}{1}", 10, 20, i=100, j=0) self.assertRaises(ValueError, fmt.format, "{0}", 10, 20) self.assertRaises(ValueError, fmt.format, "{0}", 10, 20, i=100) self.assertRaises(ValueError, fmt.format, "{i}", 10, 20, i=100) # Alternate formatting is not supported self.assertRaises(ValueError, format, '', '#') self.assertRaises(ValueError, format, '', '#20') @unittest.expectedFailure def test_format_keyword_arguments(self): fmt = string.Formatter() self.assertEqual(fmt.format("-{arg}-", arg='test'), '-test-') self.assertRaises(KeyError, fmt.format, "-{arg}-") self.assertEqual(fmt.format("-{self}-", self='test'), '-test-') self.assertRaises(KeyError, fmt.format, "-{self}-") self.assertEqual(fmt.format("-{format_string}-", format_string='test'), '-test-') self.assertRaises(KeyError, fmt.format, "-{format_string}-") self.assertEqual(fmt.format(arg='test', format_string="-{arg}-"), '-test-') class BytesAliasTest(unittest.TestCase): @unittest.expectedFailure def test_builtin(self): self.assertTrue(str is bytes) def test_syntax(self): self.assertEqual(b"spam", "spam") self.assertEqual(br"egg\foo", "egg\\foo") self.assertTrue(type(b""), str) self.assertTrue(type(br""), str) # Template tests (formerly housed in test_pep292.py) class Bag(object): pass class Mapping(object): def __getitem__(self, name): obj = self for part in name.split('.'): try: obj = getattr(obj, part) except AttributeError: raise KeyError(name) return obj class TestTemplate(unittest.TestCase): def test_regular_templates(self): s = Template('$who likes to eat a bag of $what worth $$100') self.assertEqual(s.substitute(dict(who='tim', what='ham')), 'tim likes to eat a bag of ham worth $100') self.assertRaises(KeyError, s.substitute, dict(who='tim')) self.assertRaises(TypeError, Template.substitute) def test_regular_templates_with_braces(self): s = Template('$who likes ${what} for ${meal}') d = dict(who='tim', what='ham', meal='dinner') self.assertEqual(s.substitute(d), 'tim likes ham for dinner') self.assertRaises(KeyError, s.substitute, dict(who='tim', what='ham')) def test_escapes(self): eq = self.assertEqual s = Template('$who likes to eat a bag of $$what worth $$100') eq(s.substitute(dict(who='tim', what='ham')), 'tim likes to eat a bag of $what worth $100') s = Template('$who likes $$') eq(s.substitute(dict(who='tim', what='ham')), 'tim likes $') def test_percents(self): eq = self.assertEqual s = Template('%(foo)s $foo ${foo}') d = dict(foo='baz') eq(s.substitute(d), '%(foo)s baz baz') eq(s.safe_substitute(d), '%(foo)s baz baz') def test_stringification(self): eq = self.assertEqual s = Template('tim has eaten $count bags of ham today') d = dict(count=7) eq(s.substitute(d), 'tim has eaten 7 bags of ham today') eq(s.safe_substitute(d), 'tim has eaten 7 bags of ham today') s = Template('tim has eaten ${count} bags of ham today') eq(s.substitute(d), 'tim has eaten 7 bags of ham today') def test_tupleargs(self): eq = self.assertEqual s = Template('$who ate ${meal}') d = dict(who=('tim', 'fred'), meal=('ham', 'kung pao')) eq(s.substitute(d), "('tim', 'fred') ate ('ham', 'kung pao')") eq(s.safe_substitute(d), "('tim', 'fred') ate ('ham', 'kung pao')") def test_SafeTemplate(self): eq = self.assertEqual s = Template('$who likes ${what} for ${meal}') eq(s.safe_substitute(dict(who='tim')), 'tim likes ${what} for ${meal}') eq(s.safe_substitute(dict(what='ham')), '$who likes ham for ${meal}') eq(s.safe_substitute(dict(what='ham', meal='dinner')), '$who likes ham for dinner') eq(s.safe_substitute(dict(who='tim', what='ham')), 'tim likes ham for ${meal}') eq(s.safe_substitute(dict(who='tim', what='ham', meal='dinner')), 'tim likes ham for dinner') @unittest.expectedFailure def test_invalid_placeholders(self): raises = self.assertRaises s = Template('$who likes $') raises(ValueError, s.substitute, dict(who='tim')) s = Template('$who likes ${what)') raises(ValueError, s.substitute, dict(who='tim')) s = Template('$who likes $100') raises(ValueError, s.substitute, dict(who='tim')) def test_idpattern_override(self): class PathPattern(Template): idpattern = r'[_a-z][._a-z0-9]*' m = Mapping() m.bag = Bag() m.bag.foo = Bag() m.bag.foo.who = 'tim' m.bag.what = 'ham' s = PathPattern('$bag.foo.who likes to eat a bag of $bag.what') self.assertEqual(s.substitute(m), 'tim likes to eat a bag of ham') def test_pattern_override(self): class MyPattern(Template): pattern = r""" (?P@{2}) | @(?P[_a-z][._a-z0-9]*) | @{(?P[_a-z][._a-z0-9]*)} | (?P@) """ m = Mapping() m.bag = Bag() m.bag.foo = Bag() m.bag.foo.who = 'tim' m.bag.what = 'ham' s = MyPattern('@bag.foo.who likes to eat a bag of @bag.what') self.assertEqual(s.substitute(m), 'tim likes to eat a bag of ham') class BadPattern(Template): pattern = r""" (?P.*) | (?P@{2}) | @(?P[_a-z][._a-z0-9]*) | @{(?P[_a-z][._a-z0-9]*)} | (?P@) | """ s = BadPattern('@bag.foo.who likes to eat a bag of @bag.what') self.assertRaises(ValueError, s.substitute, {}) self.assertRaises(ValueError, s.safe_substitute, {}) def test_braced_override(self): class MyTemplate(Template): pattern = r""" \$(?: (?P$) | (?P[_a-z][_a-z0-9]*) | @@(?P[_a-z][_a-z0-9]*)@@ | (?P) | ) """ tmpl = 'PyCon in $@@location@@' t = MyTemplate(tmpl) self.assertRaises(KeyError, t.substitute, {}) val = t.substitute({'location': 'Cleveland'}) self.assertEqual(val, 'PyCon in Cleveland') def test_braced_override_safe(self): class MyTemplate(Template): pattern = r""" \$(?: (?P$) | (?P[_a-z][_a-z0-9]*) | @@(?P[_a-z][_a-z0-9]*)@@ | (?P) | ) """ tmpl = 'PyCon in $@@location@@' t = MyTemplate(tmpl) self.assertEqual(t.safe_substitute(), tmpl) val = t.safe_substitute({'location': 'Cleveland'}) self.assertEqual(val, 'PyCon in Cleveland') def test_unicode_values(self): s = Template('$who likes $what') d = dict(who=u't\xffm', what=u'f\xfe\fed') self.assertEqual(s.substitute(d), u't\xffm likes f\xfe\x0ced') def test_keyword_arguments(self): eq = self.assertEqual s = Template('$who likes $what') eq(s.substitute(who='tim', what='ham'), 'tim likes ham') eq(s.substitute(dict(who='tim'), what='ham'), 'tim likes ham') eq(s.substitute(dict(who='fred', what='kung pao'), who='tim', what='ham'), 'tim likes ham') s = Template('the mapping is $mapping') eq(s.substitute(dict(foo='none'), mapping='bozo'), 'the mapping is bozo') eq(s.substitute(dict(mapping='one'), mapping='two'), 'the mapping is two') s = Template('the self is $self') eq(s.substitute(self='bozo'), 'the self is bozo') def test_keyword_arguments_safe(self): eq = self.assertEqual raises = self.assertRaises s = Template('$who likes $what') eq(s.safe_substitute(who='tim', what='ham'), 'tim likes ham') eq(s.safe_substitute(dict(who='tim'), what='ham'), 'tim likes ham') eq(s.safe_substitute(dict(who='fred', what='kung pao'), who='tim', what='ham'), 'tim likes ham') s = Template('the mapping is $mapping') eq(s.safe_substitute(dict(foo='none'), mapping='bozo'), 'the mapping is bozo') eq(s.safe_substitute(dict(mapping='one'), mapping='two'), 'the mapping is two') d = dict(mapping='one') raises(TypeError, s.substitute, d, {}) raises(TypeError, s.safe_substitute, d, {}) s = Template('the self is $self') eq(s.safe_substitute(self='bozo'), 'the self is bozo') def test_delimiter_override(self): eq = self.assertEqual raises = self.assertRaises class AmpersandTemplate(Template): delimiter = '&' s = AmpersandTemplate('this &gift is for &{who} &&') eq(s.substitute(gift='bud', who='you'), 'this bud is for you &') raises(KeyError, s.substitute) eq(s.safe_substitute(gift='bud', who='you'), 'this bud is for you &') eq(s.safe_substitute(), 'this &gift is for &{who} &') s = AmpersandTemplate('this &gift is for &{who} &') raises(ValueError, s.substitute, dict(gift='bud', who='you')) eq(s.safe_substitute(), 'this &gift is for &{who} &') class PieDelims(Template): delimiter = '@' s = PieDelims('@who likes to eat a bag of @{what} worth $100') self.assertEqual(s.substitute(dict(who='tim', what='ham')), 'tim likes to eat a bag of ham worth $100') def test_main(): test_support.run_unittest(StringTest, ModuleTest, BytesAliasTest, TestTemplate) if __name__ == '__main__': test_main() ================================================ FILE: third_party/stdlib/test/test_support.py ================================================ """Supporting definitions for the Python regression tests.""" # if __name__ != 'test.test_support': # raise ImportError('test_support must be imported from the test package') import contextlib # import errno # import functools # import gc # import socket import sys import os # import platform # import shutil import warnings import unittest # import importlib import UserDict # import re # import time # import struct # import sysconfig try: import thread except ImportError: thread = None __all__ = [ "Error", "TestFailed", "have_unicode", "BasicTestRunner", "run_unittest", "check_warnings", "check_py3k_warnings", "CleanImport", "EnvironmentVarGuard" ] # __all__ = ["Error", "TestFailed", "ResourceDenied", "import_module", # "verbose", "use_resources", "max_memuse", "record_original_stdout", # "get_original_stdout", "unload", "unlink", "rmtree", "forget", # "is_resource_enabled", "requires", "requires_mac_ver", # "find_unused_port", "bind_port", # "fcmp", "have_unicode", "is_jython", "TESTFN", "HOST", "FUZZ", # "SAVEDCWD", "temp_cwd", "findfile", "sortdict", "check_syntax_error", # "open_urlresource", "check_warnings", "check_py3k_warnings", # "CleanImport", "EnvironmentVarGuard", "captured_output", # "captured_stdout", "TransientResource", "transient_internet", # "run_with_locale", "set_memlimit", "bigmemtest", "bigaddrspacetest", # "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup", # "threading_cleanup", "reap_threads", "start_threads", "cpython_only", # "check_impl_detail", "get_attribute", "py3k_bytes", # "import_fresh_module", "threading_cleanup", "reap_children", # "strip_python_stderr", "IPV6_ENABLED", "run_with_tz"] class Error(Exception): """Base class for regression test exceptions.""" class TestFailed(Error): """Test failed.""" # class ResourceDenied(unittest.SkipTest): # """Test skipped because it requested a disallowed resource. # This is raised when a test calls requires() for a resource that # has not been enabled. It is used to distinguish between expected # and unexpected skips. # """ # @contextlib.contextmanager # def _ignore_deprecated_imports(ignore=True): # """Context manager to suppress package and module deprecation # warnings when importing them. # If ignore is False, this context manager has no effect.""" # if ignore: # with warnings.catch_warnings(): # warnings.filterwarnings("ignore", ".+ (module|package)", # DeprecationWarning) # yield # else: # yield # def import_module(name, deprecated=False): # """Import and return the module to be tested, raising SkipTest if # it is not available. # If deprecated is True, any module or package deprecation messages # will be suppressed.""" # with _ignore_deprecated_imports(deprecated): # try: # return importlib.import_module(name) # except ImportError, msg: # raise unittest.SkipTest(str(msg)) # def _save_and_remove_module(name, orig_modules): # """Helper function to save and remove a module from sys.modules # Raise ImportError if the module can't be imported.""" # # try to import the module and raise an error if it can't be imported # if name not in sys.modules: # __import__(name) # del sys.modules[name] # for modname in list(sys.modules): # if modname == name or modname.startswith(name + '.'): # orig_modules[modname] = sys.modules[modname] # del sys.modules[modname] # def _save_and_block_module(name, orig_modules): # """Helper function to save and block a module in sys.modules # Return True if the module was in sys.modules, False otherwise.""" # saved = True # try: # orig_modules[name] = sys.modules[name] # except KeyError: # saved = False # sys.modules[name] = None # return saved # def import_fresh_module(name, fresh=(), blocked=(), deprecated=False): # """Imports and returns a module, deliberately bypassing the sys.modules cache # and importing a fresh copy of the module. Once the import is complete, # the sys.modules cache is restored to its original state. # Modules named in fresh are also imported anew if needed by the import. # If one of these modules can't be imported, None is returned. # Importing of modules named in blocked is prevented while the fresh import # takes place. # If deprecated is True, any module or package deprecation messages # will be suppressed.""" # # NOTE: test_heapq, test_json, and test_warnings include extra sanity # # checks to make sure that this utility function is working as expected # with _ignore_deprecated_imports(deprecated): # # Keep track of modules saved for later restoration as well # # as those which just need a blocking entry removed # orig_modules = {} # names_to_remove = [] # _save_and_remove_module(name, orig_modules) # try: # for fresh_name in fresh: # _save_and_remove_module(fresh_name, orig_modules) # for blocked_name in blocked: # if not _save_and_block_module(blocked_name, orig_modules): # names_to_remove.append(blocked_name) # fresh_module = importlib.import_module(name) # except ImportError: # fresh_module = None # finally: # for orig_name, module in orig_modules.items(): # sys.modules[orig_name] = module # for name_to_remove in names_to_remove: # del sys.modules[name_to_remove] # return fresh_module # def get_attribute(obj, name): # """Get an attribute, raising SkipTest if AttributeError is raised.""" # try: # attribute = getattr(obj, name) # except AttributeError: # raise unittest.SkipTest("module %s has no attribute %s" % ( # obj.__name__, name)) # else: # return attribute verbose = 1 # Flag set to 0 by regrtest.py # use_resources = None # Flag set to [] by regrtest.py # max_memuse = 0 # Disable bigmem tests (they will still be run with # # small sizes, to make sure they work.) # real_max_memuse = 0 # # _original_stdout is meant to hold stdout at the time regrtest began. # # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. # # The point is to have some flavor of stdout the user can actually see. # _original_stdout = None # def record_original_stdout(stdout): # global _original_stdout # _original_stdout = stdout # def get_original_stdout(): # return _original_stdout or sys.stdout # def unload(name): # try: # del sys.modules[name] # except KeyError: # pass if sys.platform.startswith("win"): def _waitfor(func, pathname, waitall=False): # Perform the operation func(pathname) # Now setup the wait loop if waitall: dirname = pathname else: dirname, name = os.path.split(pathname) dirname = dirname or '.' # Check for `pathname` to be removed from the filesystem. # The exponential backoff of the timeout amounts to a total # of ~1 second after which the deletion is probably an error # anyway. # Testing on a i7@4.3GHz shows that usually only 1 iteration is # required when contention occurs. timeout = 0.001 while timeout < 1.0: # Note we are only testing for the existence of the file(s) in # the contents of the directory regardless of any security or # access rights. If we have made it this far, we have sufficient # permissions to do that much using Python's equivalent of the # Windows API FindFirstFile. # Other Windows APIs can fail or give incorrect results when # dealing with files that are pending deletion. L = os.listdir(dirname) if not (L if waitall else name in L): return # Increase the timeout and try again time.sleep(timeout) timeout *= 2 warnings.warn('tests may fail, delete still pending for ' + pathname, RuntimeWarning, stacklevel=4) def _unlink(filename): _waitfor(os.unlink, filename) def _rmdir(dirname): _waitfor(os.rmdir, dirname) def _rmtree(path): def _rmtree_inner(path): for name in os.listdir(path): fullname = os.path.join(path, name) if os.path.isdir(fullname): _waitfor(_rmtree_inner, fullname, waitall=True) os.rmdir(fullname) else: os.unlink(fullname) _waitfor(_rmtree_inner, path, waitall=True) _waitfor(os.rmdir, path) else: _unlink = os.unlink _rmdir = os.rmdir # _rmtree = shutil.rmtree def unlink(filename): try: _unlink(filename) except OSError: pass # def rmdir(dirname): # try: # _rmdir(dirname) # except OSError as error: # # The directory need not exist. # if error.errno != errno.ENOENT: # raise # def rmtree(path): # try: # _rmtree(path) # except OSError, e: # # Unix returns ENOENT, Windows returns ESRCH. # if e.errno not in (errno.ENOENT, errno.ESRCH): # raise # def forget(modname): # '''"Forget" a module was ever imported by removing it from sys.modules and # deleting any .pyc and .pyo files.''' # unload(modname) # for dirname in sys.path: # unlink(os.path.join(dirname, modname + os.extsep + 'pyc')) # # Deleting the .pyo file cannot be within the 'try' for the .pyc since # # the chance exists that there is no .pyc (and thus the 'try' statement # # is exited) but there is a .pyo file. # unlink(os.path.join(dirname, modname + os.extsep + 'pyo')) # # Check whether a gui is actually available # def _is_gui_available(): # if hasattr(_is_gui_available, 'result'): # return _is_gui_available.result # reason = None # if sys.platform.startswith('win'): # # if Python is running as a service (such as the buildbot service), # # gui interaction may be disallowed # import ctypes # import ctypes.wintypes # UOI_FLAGS = 1 # WSF_VISIBLE = 0x0001 # class USEROBJECTFLAGS(ctypes.Structure): # _fields_ = [("fInherit", ctypes.wintypes.BOOL), # ("fReserved", ctypes.wintypes.BOOL), # ("dwFlags", ctypes.wintypes.DWORD)] # dll = ctypes.windll.user32 # h = dll.GetProcessWindowStation() # if not h: # raise ctypes.WinError() # uof = USEROBJECTFLAGS() # needed = ctypes.wintypes.DWORD() # res = dll.GetUserObjectInformationW(h, # UOI_FLAGS, # ctypes.byref(uof), # ctypes.sizeof(uof), # ctypes.byref(needed)) # if not res: # raise ctypes.WinError() # if not bool(uof.dwFlags & WSF_VISIBLE): # reason = "gui not available (WSF_VISIBLE flag not set)" # elif sys.platform == 'darwin': # # The Aqua Tk implementations on OS X can abort the process if # # being called in an environment where a window server connection # # cannot be made, for instance when invoked by a buildbot or ssh # # process not running under the same user id as the current console # # user. To avoid that, raise an exception if the window manager # # connection is not available. # from ctypes import cdll, c_int, pointer, Structure # from ctypes.util import find_library # app_services = cdll.LoadLibrary(find_library("ApplicationServices")) # if app_services.CGMainDisplayID() == 0: # reason = "gui tests cannot run without OS X window manager" # else: # class ProcessSerialNumber(Structure): # _fields_ = [("highLongOfPSN", c_int), # ("lowLongOfPSN", c_int)] # psn = ProcessSerialNumber() # psn_p = pointer(psn) # if ( (app_services.GetCurrentProcess(psn_p) < 0) or # (app_services.SetFrontProcess(psn_p) < 0) ): # reason = "cannot run without OS X gui process" # # check on every platform whether tkinter can actually do anything # if not reason: # try: # from Tkinter import Tk # root = Tk() # root.update() # root.destroy() # except Exception as e: # err_string = str(e) # if len(err_string) > 50: # err_string = err_string[:50] + ' [...]' # reason = 'Tk unavailable due to {}: {}'.format(type(e).__name__, # err_string) # _is_gui_available.reason = reason # _is_gui_available.result = not reason # return _is_gui_available.result # def is_resource_enabled(resource): # """Test whether a resource is enabled. # Known resources are set by regrtest.py. If not running under regrtest.py, # all resources are assumed enabled unless use_resources has been set. # """ # return use_resources is None or resource in use_resources # def requires(resource, msg=None): # """Raise ResourceDenied if the specified resource is not available.""" # if resource == 'gui' and not _is_gui_available(): # raise ResourceDenied(_is_gui_available.reason) # if not is_resource_enabled(resource): # if msg is None: # msg = "Use of the `%s' resource not enabled" % resource # raise ResourceDenied(msg) # def requires_mac_ver(*min_version): # """Decorator raising SkipTest if the OS is Mac OS X and the OS X # version if less than min_version. # For example, @requires_mac_ver(10, 5) raises SkipTest if the OS X version # is lesser than 10.5. # """ # def decorator(func): # @functools.wraps(func) # def wrapper(*args, **kw): # if sys.platform == 'darwin': # version_txt = platform.mac_ver()[0] # try: # version = tuple(map(int, version_txt.split('.'))) # except ValueError: # pass # else: # if version < min_version: # min_version_txt = '.'.join(map(str, min_version)) # raise unittest.SkipTest( # "Mac OS X %s or higher required, not %s" # % (min_version_txt, version_txt)) # return func(*args, **kw) # wrapper.min_version = min_version # return wrapper # return decorator # # Don't use "localhost", since resolving it uses the DNS under recent # # Windows versions (see issue #18792). # HOST = "127.0.0.1" # HOSTv6 = "::1" # def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): # """Returns an unused port that should be suitable for binding. This is # achieved by creating a temporary socket with the same family and type as # the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to # the specified host address (defaults to 0.0.0.0) with the port set to 0, # eliciting an unused ephemeral port from the OS. The temporary socket is # then closed and deleted, and the ephemeral port is returned. # Either this method or bind_port() should be used for any tests where a # server socket needs to be bound to a particular port for the duration of # the test. Which one to use depends on whether the calling code is creating # a python socket, or if an unused port needs to be provided in a constructor # or passed to an external program (i.e. the -accept argument to openssl's # s_server mode). Always prefer bind_port() over find_unused_port() where # possible. Hard coded ports should *NEVER* be used. As soon as a server # socket is bound to a hard coded port, the ability to run multiple instances # of the test simultaneously on the same host is compromised, which makes the # test a ticking time bomb in a buildbot environment. On Unix buildbots, this # may simply manifest as a failed test, which can be recovered from without # intervention in most cases, but on Windows, the entire python process can # completely and utterly wedge, requiring someone to log in to the buildbot # and manually kill the affected process. # (This is easy to reproduce on Windows, unfortunately, and can be traced to # the SO_REUSEADDR socket option having different semantics on Windows versus # Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind, # listen and then accept connections on identical host/ports. An EADDRINUSE # socket.error will be raised at some point (depending on the platform and # the order bind and listen were called on each socket). # However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE # will ever be raised when attempting to bind two identical host/ports. When # accept() is called on each socket, the second caller's process will steal # the port from the first caller, leaving them both in an awkwardly wedged # state where they'll no longer respond to any signals or graceful kills, and # must be forcibly killed via OpenProcess()/TerminateProcess(). # The solution on Windows is to use the SO_EXCLUSIVEADDRUSE socket option # instead of SO_REUSEADDR, which effectively affords the same semantics as # SO_REUSEADDR on Unix. Given the propensity of Unix developers in the Open # Source world compared to Windows ones, this is a common mistake. A quick # look over OpenSSL's 0.9.8g source shows that they use SO_REUSEADDR when # openssl.exe is called with the 's_server' option, for example. See # http://bugs.python.org/issue2550 for more info. The following site also # has a very thorough description about the implications of both REUSEADDR # and EXCLUSIVEADDRUSE on Windows: # http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx) # XXX: although this approach is a vast improvement on previous attempts to # elicit unused ports, it rests heavily on the assumption that the ephemeral # port returned to us by the OS won't immediately be dished back out to some # other process when we close and delete our temporary socket but before our # calling code has a chance to bind the returned port. We can deal with this # issue if/when we come across it.""" # tempsock = socket.socket(family, socktype) # port = bind_port(tempsock) # tempsock.close() # del tempsock # return port # def bind_port(sock, host=HOST): # """Bind the socket to a free port and return the port number. Relies on # ephemeral ports in order to ensure we are using an unbound port. This is # important as many tests may be running simultaneously, especially in a # buildbot environment. This method raises an exception if the sock.family # is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR # or SO_REUSEPORT set on it. Tests should *never* set these socket options # for TCP/IP sockets. The only case for setting these options is testing # multicasting via multiple UDP sockets. # Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e. # on Windows), it will be set on the socket. This will prevent anyone else # from bind()'ing to our host/port for the duration of the test. # """ # if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM: # if hasattr(socket, 'SO_REUSEADDR'): # if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1: # raise TestFailed("tests should never set the SO_REUSEADDR " \ # "socket option on TCP/IP sockets!") # if hasattr(socket, 'SO_REUSEPORT'): # try: # if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1: # raise TestFailed("tests should never set the SO_REUSEPORT " \ # "socket option on TCP/IP sockets!") # except EnvironmentError: # # Python's socket module was compiled using modern headers # # thus defining SO_REUSEPORT but this process is running # # under an older kernel that does not support SO_REUSEPORT. # pass # if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): # sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) # sock.bind((host, 0)) # port = sock.getsockname()[1] # return port # def _is_ipv6_enabled(): # """Check whether IPv6 is enabled on this host.""" # if socket.has_ipv6: # sock = None # try: # sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) # sock.bind((HOSTv6, 0)) # return True # except socket.error: # pass # finally: # if sock: # sock.close() # return False # IPV6_ENABLED = _is_ipv6_enabled() # def system_must_validate_cert(f): # """Skip the test on TLS certificate validation failures.""" # @functools.wraps(f) # def dec(*args, **kwargs): # try: # f(*args, **kwargs) # except IOError as e: # if "CERTIFICATE_VERIFY_FAILED" in str(e): # raise unittest.SkipTest("system does not contain " # "necessary certificates") # raise # return dec # FUZZ = 1e-6 # def fcmp(x, y): # fuzzy comparison function # if isinstance(x, float) or isinstance(y, float): # try: # fuzz = (abs(x) + abs(y)) * FUZZ # if abs(x-y) <= fuzz: # return 0 # except: # pass # elif type(x) == type(y) and isinstance(x, (tuple, list)): # for i in range(min(len(x), len(y))): # outcome = fcmp(x[i], y[i]) # if outcome != 0: # return outcome # return (len(x) > len(y)) - (len(x) < len(y)) # return (x > y) - (x < y) # # A constant likely larger than the underlying OS pipe buffer size, to # # make writes blocking. # # Windows limit seems to be around 512 B, and many Unix kernels have a # # 64 KiB pipe buffer size or 16 * PAGE_SIZE: take a few megs to be sure. # # (see issue #17835 for a discussion of this number). # PIPE_MAX_SIZE = 4 * 1024 * 1024 + 1 # # A constant likely larger than the underlying OS socket buffer size, to make # # writes blocking. # # The socket buffer sizes can usually be tuned system-wide (e.g. through sysctl # # on Linux), or on a per-socket basis (SO_SNDBUF/SO_RCVBUF). See issue #18643 # # for a discussion of this number). # SOCK_MAX_SIZE = 16 * 1024 * 1024 + 1 # is_jython = sys.platform.startswith('java') try: unicode have_unicode = True except NameError: have_unicode = False requires_unicode = unittest.skipUnless(have_unicode, 'no unicode support') # def u(s): # return unicode(s, 'unicode-escape') # FS_NONASCII: non-ASCII Unicode character encodable by # sys.getfilesystemencoding(), or None if there is no such character. FS_NONASCII = None # if have_unicode: # for character in ( # # First try printable and common characters to have a readable filename. # # For each character, the encoding list are just example of encodings able # # to encode the character (the list is not exhaustive). # # U+00E6 (Latin Small Letter Ae): cp1252, iso-8859-1 # unichr(0x00E6), # # U+0130 (Latin Capital Letter I With Dot Above): cp1254, iso8859_3 # unichr(0x0130), # # U+0141 (Latin Capital Letter L With Stroke): cp1250, cp1257 # unichr(0x0141), # # U+03C6 (Greek Small Letter Phi): cp1253 # unichr(0x03C6), # # U+041A (Cyrillic Capital Letter Ka): cp1251 # unichr(0x041A), # # U+05D0 (Hebrew Letter Alef): Encodable to cp424 # unichr(0x05D0), # # U+060C (Arabic Comma): cp864, cp1006, iso8859_6, mac_arabic # unichr(0x060C), # # U+062A (Arabic Letter Teh): cp720 # unichr(0x062A), # # U+0E01 (Thai Character Ko Kai): cp874 # unichr(0x0E01), # # Then try more "special" characters. "special" because they may be # # interpreted or displayed differently depending on the exact locale # # encoding and the font. # # U+00A0 (No-Break Space) # unichr(0x00A0), # # U+20AC (Euro Sign) # unichr(0x20AC), # ): # try: # character.encode(sys.getfilesystemencoding())\ # .decode(sys.getfilesystemencoding()) # except UnicodeError: # pass # else: # FS_NONASCII = character # break # Filename used for testing if os.name == 'java': # Jython disallows @ in module names TESTFN = '$test' elif os.name == 'riscos': TESTFN = 'testfile' else: TESTFN = '@test' # # Unicode name only used if TEST_FN_ENCODING exists for the platform. # if have_unicode: # # Assuming sys.getfilesystemencoding()!=sys.getdefaultencoding() # # TESTFN_UNICODE is a filename that can be encoded using the # # file system encoding, but *not* with the default (ascii) encoding # if isinstance('', unicode): # # python -U # # XXX perhaps unicode() should accept Unicode strings? # TESTFN_UNICODE = "@test-\xe0\xf2" # else: # # 2 latin characters. # TESTFN_UNICODE = unicode("@test-\xe0\xf2", "latin-1") # TESTFN_ENCODING = sys.getfilesystemencoding() # # TESTFN_UNENCODABLE is a filename that should *not* be # # able to be encoded by *either* the default or filesystem encoding. # # This test really only makes sense on Windows NT platforms # # which have special Unicode support in posixmodule. # if (not hasattr(sys, "getwindowsversion") or # sys.getwindowsversion()[3] < 2): # 0=win32s or 1=9x/ME # TESTFN_UNENCODABLE = None # else: # # Japanese characters (I think - from bug 846133) # TESTFN_UNENCODABLE = eval('u"@test-\u5171\u6709\u3055\u308c\u308b"') # try: # # XXX - Note - should be using TESTFN_ENCODING here - but for # # Windows, "mbcs" currently always operates as if in # # errors=ignore' mode - hence we get '?' characters rather than # # the exception. 'Latin1' operates as we expect - ie, fails. # # See [ 850997 ] mbcs encoding ignores errors # TESTFN_UNENCODABLE.encode("Latin1") # except UnicodeEncodeError: # pass # else: # print \ # 'WARNING: The filename %r CAN be encoded by the filesystem. ' \ # 'Unicode filename tests may not be effective' \ # % TESTFN_UNENCODABLE # Disambiguate TESTFN for parallel testing, while letting it remain a valid # module name. TESTFN = "%s_%s_tmp" % (TESTFN, os.getpid()) # # Save the initial cwd # SAVEDCWD = os.getcwd() # @contextlib.contextmanager # def change_cwd(path, quiet=False): # """Return a context manager that changes the current working directory. # Arguments: # path: the directory to use as the temporary current working directory. # quiet: if False (the default), the context manager raises an exception # on error. Otherwise, it issues only a warning and keeps the current # working directory the same. # """ # saved_dir = os.getcwd() # try: # os.chdir(path) # except OSError: # if not quiet: # raise # warnings.warn('tests may fail, unable to change CWD to: ' + path, # RuntimeWarning, stacklevel=3) # try: # yield os.getcwd() # finally: # os.chdir(saved_dir) # @contextlib.contextmanager # def temp_cwd(name='tempcwd', quiet=False): # """ # Context manager that creates a temporary directory and set it as CWD. # The new CWD is created in the current directory and it's named *name*. # If *quiet* is False (default) and it's not possible to create or change # the CWD, an error is raised. If it's True, only a warning is raised # and the original CWD is used. # """ # if (have_unicode and isinstance(name, unicode) and # not os.path.supports_unicode_filenames): # try: # name = name.encode(sys.getfilesystemencoding() or 'ascii') # except UnicodeEncodeError: # if not quiet: # raise unittest.SkipTest('unable to encode the cwd name with ' # 'the filesystem encoding.') # saved_dir = os.getcwd() # is_temporary = False # try: # os.mkdir(name) # os.chdir(name) # is_temporary = True # except OSError: # if not quiet: # raise # warnings.warn('tests may fail, unable to change the CWD to ' + name, # RuntimeWarning, stacklevel=3) # try: # yield os.getcwd() # finally: # os.chdir(saved_dir) # if is_temporary: # rmtree(name) # def findfile(file, here=__file__, subdir=None): # """Try to find a file on sys.path and the working directory. If it is not # found the argument passed to the function is returned (this does not # necessarily signal failure; could still be the legitimate path).""" # if os.path.isabs(file): # return file # if subdir is not None: # file = os.path.join(subdir, file) # path = sys.path # path = [os.path.dirname(here)] + path # for dn in path: # fn = os.path.join(dn, file) # if os.path.exists(fn): return fn # return file # def sortdict(dict): # "Like repr(dict), but in sorted order." # items = dict.items() # items.sort() # reprpairs = ["%r: %r" % pair for pair in items] # withcommas = ", ".join(reprpairs) # return "{%s}" % withcommas # def make_bad_fd(): # """ # Create an invalid file descriptor by opening and closing a file and return # its fd. # """ # file = open(TESTFN, "wb") # try: # return file.fileno() # finally: # file.close() # unlink(TESTFN) # def check_syntax_error(testcase, statement): # testcase.assertRaises(SyntaxError, compile, statement, # '', 'exec') # def open_urlresource(url, check=None): # import urlparse, urllib2 # filename = urlparse.urlparse(url)[2].split('/')[-1] # '/': it's URL! # fn = os.path.join(os.path.dirname(__file__), "data", filename) # def check_valid_file(fn): # f = open(fn) # if check is None: # return f # elif check(f): # f.seek(0) # return f # f.close() # if os.path.exists(fn): # f = check_valid_file(fn) # if f is not None: # return f # unlink(fn) # # Verify the requirement before downloading the file # requires('urlfetch') # print >> get_original_stdout(), '\tfetching %s ...' % url # f = urllib2.urlopen(url, timeout=15) # try: # with open(fn, "wb") as out: # s = f.read() # while s: # out.write(s) # s = f.read() # finally: # f.close() # f = check_valid_file(fn) # if f is not None: # return f # raise TestFailed('invalid resource "%s"' % fn) class WarningsRecorder(object): """Convenience wrapper for the warnings list returned on entry to the warnings.catch_warnings() context manager. """ def __init__(self, warnings_list): self._warnings = warnings_list self._last = 0 def __getattr__(self, attr): if len(self._warnings) > self._last: return getattr(self._warnings[-1], attr) elif attr in warnings.WarningMessage._WARNING_DETAILS: return None raise AttributeError("%r has no attribute %r" % (self, attr)) @property def warnings(self): return self._warnings[self._last:] def reset(self): self._last = len(self._warnings) def _filterwarnings(filters, quiet=False): """Catch the warnings, then check if all the expected warnings have been raised and re-raise unexpected warnings. If 'quiet' is True, only re-raise the unexpected warnings. """ # Clear the warning registry of the calling module # in order to re-raise the warnings. # frame = sys._getframe(2) # registry = frame.f_globals.get('__warningregistry__') # if registry: # registry.clear() with warnings.catch_warnings(record=True) as w: # Set filter "always" to record all warnings. Because # test_warnings swap the module, we need to look up in # the sys.modules dictionary. sys.modules['warnings'].simplefilter("always") yield WarningsRecorder(w) # Filter the recorded warnings reraise = [warning.message for warning in w] missing = [] for msg, cat in filters: seen = False for exc in reraise[:]: message = str(exc) # Filter out the matching messages if (re.match(msg, message, re.I) and issubclass(exc.__class__, cat)): seen = True reraise.remove(exc) if not seen and not quiet: # This filter caught nothing missing.append((msg, cat.__name__)) if reraise: raise AssertionError("unhandled warning %r" % reraise[0]) if missing: raise AssertionError("filter (%r, %s) did not catch any warning" % missing[0]) @contextlib.contextmanager def check_warnings(*filters, **kwargs): """Context manager to silence warnings. Accept 2-tuples as positional arguments: ("message regexp", WarningCategory) Optional argument: - if 'quiet' is True, it does not fail if a filter catches nothing (default True without argument, default False if some filters are defined) Without argument, it defaults to: check_warnings(("", Warning), quiet=True) """ quiet = kwargs.get('quiet') if not filters: filters = (("", Warning),) # Preserve backward compatibility if quiet is None: quiet = True return _filterwarnings(filters, quiet) @contextlib.contextmanager def check_py3k_warnings(*filters, **kwargs): """Context manager to silence py3k warnings. Accept 2-tuples as positional arguments: ("message regexp", WarningCategory) Optional argument: - if 'quiet' is True, it does not fail if a filter catches nothing (default False) Without argument, it defaults to: check_py3k_warnings(("", DeprecationWarning), quiet=False) """ if sys.py3kwarning: if not filters: filters = (("", DeprecationWarning),) else: # It should not raise any py3k warning filters = () return _filterwarnings(filters, kwargs.get('quiet')) class CleanImport(object): """Context manager to force import to return a new module reference. This is useful for testing module-level behaviours, such as the emission of a DeprecationWarning on import. Use like this: with CleanImport("foo"): importlib.import_module("foo") # new reference """ def __init__(self, *module_names): self.original_modules = sys.modules.copy() for module_name in module_names: if module_name in sys.modules: module = sys.modules[module_name] # It is possible that module_name is just an alias for # another module (e.g. stub for modules renamed in 3.x). # In that case, we also need delete the real module to clear # the import cache. if module.__name__ != module_name: del sys.modules[module.__name__] del sys.modules[module_name] def __enter__(self): return self def __exit__(self, *ignore_exc): sys.modules.update(self.original_modules) class EnvironmentVarGuard(UserDict.DictMixin): """Class to help protect the environment variable properly. Can be used as a context manager.""" def __init__(self): self._environ = os.environ self._changed = {} def __getitem__(self, envvar): return self._environ[envvar] def __setitem__(self, envvar, value): # Remember the initial value on the first access if envvar not in self._changed: self._changed[envvar] = self._environ.get(envvar) self._environ[envvar] = value def __delitem__(self, envvar): # Remember the initial value on the first access if envvar not in self._changed: self._changed[envvar] = self._environ.get(envvar) if envvar in self._environ: del self._environ[envvar] def keys(self): return self._environ.keys() def set(self, envvar, value): self[envvar] = value def unset(self, envvar): del self[envvar] def __enter__(self): return self def __exit__(self, *ignore_exc): for (k, v) in self._changed.items(): if v is None: if k in self._environ: del self._environ[k] else: self._environ[k] = v os.environ = self._environ # class DirsOnSysPath(object): # """Context manager to temporarily add directories to sys.path. # This makes a copy of sys.path, appends any directories given # as positional arguments, then reverts sys.path to the copied # settings when the context ends. # Note that *all* sys.path modifications in the body of the # context manager, including replacement of the object, # will be reverted at the end of the block. # """ # def __init__(self, *paths): # self.original_value = sys.path[:] # self.original_object = sys.path # sys.path.extend(paths) # def __enter__(self): # return self # def __exit__(self, *ignore_exc): # sys.path = self.original_object # sys.path[:] = self.original_value # class TransientResource(object): # """Raise ResourceDenied if an exception is raised while the context manager # is in effect that matches the specified exception and attributes.""" # def __init__(self, exc, **kwargs): # self.exc = exc # self.attrs = kwargs # def __enter__(self): # return self # def __exit__(self, type_=None, value=None, traceback=None): # """If type_ is a subclass of self.exc and value has attributes matching # self.attrs, raise ResourceDenied. Otherwise let the exception # propagate (if any).""" # if type_ is not None and issubclass(self.exc, type_): # for attr, attr_value in self.attrs.iteritems(): # if not hasattr(value, attr): # break # if getattr(value, attr) != attr_value: # break # else: # raise ResourceDenied("an optional resource is not available") # @contextlib.contextmanager # def transient_internet(resource_name, timeout=30.0, errnos=()): # """Return a context manager that raises ResourceDenied when various issues # with the Internet connection manifest themselves as exceptions.""" # default_errnos = [ # ('ECONNREFUSED', 111), # ('ECONNRESET', 104), # ('EHOSTUNREACH', 113), # ('ENETUNREACH', 101), # ('ETIMEDOUT', 110), # ] # default_gai_errnos = [ # ('EAI_AGAIN', -3), # ('EAI_FAIL', -4), # ('EAI_NONAME', -2), # ('EAI_NODATA', -5), # # Windows defines EAI_NODATA as 11001 but idiotic getaddrinfo() # # implementation actually returns WSANO_DATA i.e. 11004. # ('WSANO_DATA', 11004), # ] # denied = ResourceDenied("Resource '%s' is not available" % resource_name) # captured_errnos = errnos # gai_errnos = [] # if not captured_errnos: # captured_errnos = [getattr(errno, name, num) # for (name, num) in default_errnos] # gai_errnos = [getattr(socket, name, num) # for (name, num) in default_gai_errnos] # def filter_error(err): # n = getattr(err, 'errno', None) # if (isinstance(err, socket.timeout) or # (isinstance(err, socket.gaierror) and n in gai_errnos) or # n in captured_errnos): # if not verbose: # sys.stderr.write(denied.args[0] + "\n") # raise denied # old_timeout = socket.getdefaulttimeout() # try: # if timeout is not None: # socket.setdefaulttimeout(timeout) # yield # except IOError as err: # # urllib can wrap original socket errors multiple times (!), we must # # unwrap to get at the original error. # while True: # a = err.args # if len(a) >= 1 and isinstance(a[0], IOError): # err = a[0] # # The error can also be wrapped as args[1]: # # except socket.error as msg: # # raise IOError('socket error', msg).with_traceback(sys.exc_info()[2]) # elif len(a) >= 2 and isinstance(a[1], IOError): # err = a[1] # else: # break # filter_error(err) # raise # # XXX should we catch generic exceptions and look for their # # __cause__ or __context__? # finally: # socket.setdefaulttimeout(old_timeout) # @contextlib.contextmanager # def captured_output(stream_name): # """Return a context manager used by captured_stdout and captured_stdin # that temporarily replaces the sys stream *stream_name* with a StringIO.""" # import StringIO # orig_stdout = getattr(sys, stream_name) # setattr(sys, stream_name, StringIO.StringIO()) # try: # yield getattr(sys, stream_name) # finally: # setattr(sys, stream_name, orig_stdout) # def captured_stdout(): # """Capture the output of sys.stdout: # with captured_stdout() as s: # print "hello" # self.assertEqual(s.getvalue(), "hello") # """ # return captured_output("stdout") # def captured_stderr(): # return captured_output("stderr") # def captured_stdin(): # return captured_output("stdin") # def gc_collect(): # """Force as many objects as possible to be collected. # In non-CPython implementations of Python, this is needed because timely # deallocation is not guaranteed by the garbage collector. (Even in CPython # this can be the case in case of reference cycles.) This means that __del__ # methods may be called later than expected and weakrefs may remain alive for # longer than expected. This function tries its best to force all garbage # objects to disappear. # """ # gc.collect() # if is_jython: # time.sleep(0.1) # gc.collect() # gc.collect() # _header = '2P' # if hasattr(sys, "gettotalrefcount"): # _header = '2P' + _header # _vheader = _header + 'P' # def calcobjsize(fmt): # return struct.calcsize(_header + fmt + '0P') # def calcvobjsize(fmt): # return struct.calcsize(_vheader + fmt + '0P') # _TPFLAGS_HAVE_GC = 1<<14 # _TPFLAGS_HEAPTYPE = 1<<9 # def check_sizeof(test, o, size): # import _testcapi # result = sys.getsizeof(o) # # add GC header size # if ((type(o) == type) and (o.__flags__ & _TPFLAGS_HEAPTYPE) or\ # ((type(o) != type) and (type(o).__flags__ & _TPFLAGS_HAVE_GC))): # size += _testcapi.SIZEOF_PYGC_HEAD # msg = 'wrong size for %s: got %d, expected %d' \ # % (type(o), result, size) # test.assertEqual(result, size, msg) # #======================================================================= # # Decorator for running a function in a different locale, correctly resetting # # it afterwards. # def run_with_locale(catstr, *locales): # def decorator(func): # def inner(*args, **kwds): # try: # import locale # category = getattr(locale, catstr) # orig_locale = locale.setlocale(category) # except AttributeError: # # if the test author gives us an invalid category string # raise # except: # # cannot retrieve original locale, so do nothing # locale = orig_locale = None # else: # for loc in locales: # try: # locale.setlocale(category, loc) # break # except: # pass # # now run the function, resetting the locale on exceptions # try: # return func(*args, **kwds) # finally: # if locale and orig_locale: # locale.setlocale(category, orig_locale) # inner.func_name = func.func_name # inner.__doc__ = func.__doc__ # return inner # return decorator # #======================================================================= # # Decorator for running a function in a specific timezone, correctly # # resetting it afterwards. # def run_with_tz(tz): # def decorator(func): # def inner(*args, **kwds): # try: # tzset = time.tzset # except AttributeError: # raise unittest.SkipTest("tzset required") # if 'TZ' in os.environ: # orig_tz = os.environ['TZ'] # else: # orig_tz = None # os.environ['TZ'] = tz # tzset() # # now run the function, resetting the tz on exceptions # try: # return func(*args, **kwds) # finally: # if orig_tz is None: # del os.environ['TZ'] # else: # os.environ['TZ'] = orig_tz # time.tzset() # inner.__name__ = func.__name__ # inner.__doc__ = func.__doc__ # return inner # return decorator # #======================================================================= # # Big-memory-test support. Separate from 'resources' because memory use should be configurable. # # Some handy shorthands. Note that these are used for byte-limits as well # # as size-limits, in the various bigmem tests # _1M = 1024*1024 # _1G = 1024 * _1M # _2G = 2 * _1G # _4G = 4 * _1G MAX_Py_ssize_t = sys.maxsize # def set_memlimit(limit): # global max_memuse # global real_max_memuse # sizes = { # 'k': 1024, # 'm': _1M, # 'g': _1G, # 't': 1024*_1G, # } # m = re.match(r'(\d+(\.\d+)?) (K|M|G|T)b?$', limit, # re.IGNORECASE | re.VERBOSE) # if m is None: # raise ValueError('Invalid memory limit %r' % (limit,)) # memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()]) # real_max_memuse = memlimit # if memlimit > MAX_Py_ssize_t: # memlimit = MAX_Py_ssize_t # if memlimit < _2G - 1: # raise ValueError('Memory limit %r too low to be useful' % (limit,)) # max_memuse = memlimit # def bigmemtest(minsize, memuse, overhead=5*_1M): # """Decorator for bigmem tests. # 'minsize' is the minimum useful size for the test (in arbitrary, # test-interpreted units.) 'memuse' is the number of 'bytes per size' for # the test, or a good estimate of it. 'overhead' specifies fixed overhead, # independent of the testsize, and defaults to 5Mb. # The decorator tries to guess a good value for 'size' and passes it to # the decorated test function. If minsize * memuse is more than the # allowed memory use (as defined by max_memuse), the test is skipped. # Otherwise, minsize is adjusted upward to use up to max_memuse. # """ # def decorator(f): # def wrapper(self): # if not max_memuse: # # If max_memuse is 0 (the default), # # we still want to run the tests with size set to a few kb, # # to make sure they work. We still want to avoid using # # too much memory, though, but we do that noisily. # maxsize = 5147 # self.assertFalse(maxsize * memuse + overhead > 20 * _1M) # else: # maxsize = int((max_memuse - overhead) / memuse) # if maxsize < minsize: # # Really ought to print 'test skipped' or something # if verbose: # sys.stderr.write("Skipping %s because of memory " # "constraint\n" % (f.__name__,)) # return # # Try to keep some breathing room in memory use # maxsize = max(maxsize - 50 * _1M, minsize) # return f(self, maxsize) # wrapper.minsize = minsize # wrapper.memuse = memuse # wrapper.overhead = overhead # return wrapper # return decorator # def precisionbigmemtest(size, memuse, overhead=5*_1M, dry_run=True): # def decorator(f): # def wrapper(self): # if not real_max_memuse: # maxsize = 5147 # else: # maxsize = size # if ((real_max_memuse or not dry_run) # and real_max_memuse < maxsize * memuse): # if verbose: # sys.stderr.write("Skipping %s because of memory " # "constraint\n" % (f.__name__,)) # return # return f(self, maxsize) # wrapper.size = size # wrapper.memuse = memuse # wrapper.overhead = overhead # return wrapper # return decorator # def bigaddrspacetest(f): # """Decorator for tests that fill the address space.""" # def wrapper(self): # if max_memuse < MAX_Py_ssize_t: # if verbose: # sys.stderr.write("Skipping %s because of memory " # "constraint\n" % (f.__name__,)) # else: # return f(self) # return wrapper #======================================================================= # unittest integration. class BasicTestRunner(object): def run(self, test): result = unittest.TestResult() test(result) return result def _id(obj): return obj # def requires_resource(resource): # if resource == 'gui' and not _is_gui_available(): # return unittest.skip(_is_gui_available.reason) # if is_resource_enabled(resource): # return _id # else: # return unittest.skip("resource {0!r} is not enabled".format(resource)) def cpython_only(test): return lambda *arg, **kw: None # def cpython_only(test): # """ # Decorator for tests only applicable on CPython. # """ # return impl_detail(cpython=True)(test) # def impl_detail(msg=None, **guards): # if check_impl_detail(**guards): # return _id # if msg is None: # guardnames, default = _parse_guards(guards) # if default: # msg = "implementation detail not available on {0}" # else: # msg = "implementation detail specific to {0}" # guardnames = sorted(guardnames.keys()) # msg = msg.format(' or '.join(guardnames)) # return unittest.skip(msg) # def _parse_guards(guards): # # Returns a tuple ({platform_name: run_me}, default_value) # if not guards: # return ({'cpython': True}, False) # is_true = guards.values()[0] # assert guards.values() == [is_true] * len(guards) # all True or all False # return (guards, not is_true) # # Use the following check to guard CPython's implementation-specific tests -- # # or to run them only on the implementation(s) guarded by the arguments. # def check_impl_detail(**guards): # """This function returns True or False depending on the host platform. # Examples: # if check_impl_detail(): # only on CPython (default) # if check_impl_detail(jython=True): # only on Jython # if check_impl_detail(cpython=False): # everywhere except on CPython # """ # guards, default = _parse_guards(guards) # return guards.get(platform.python_implementation().lower(), default) def _run_suite(suite): """Run tests from a unittest.TestSuite-derived class.""" if verbose: runner = unittest.TextTestRunner(sys.stdout, verbosity=2) else: runner = BasicTestRunner() result = runner.run(suite) if not result.wasSuccessful(): if len(result.errors) == 1 and not result.failures: err = result.errors[0][1] elif len(result.failures) == 1 and not result.errors: err = result.failures[0][1] else: err = "multiple errors occurred" if not verbose: err += "; run in verbose mode for details" raise TestFailed(err) def run_unittest(*classes): """Run tests from unittest.TestCase-derived classes.""" valid_types = (unittest.TestSuite, unittest.TestCase) suite = unittest.TestSuite() for cls in classes: if isinstance(cls, str): if cls in sys.modules: suite.addTest(unittest.findTestCases(sys.modules[cls])) else: raise ValueError("str arguments must be keys in sys.modules") elif isinstance(cls, valid_types): suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) _run_suite(suite) # #======================================================================= # # Check for the presence of docstrings. # HAVE_DOCSTRINGS = (check_impl_detail(cpython=False) or # sys.platform == 'win32' or # sysconfig.get_config_var('WITH_DOC_STRINGS')) # requires_docstrings = unittest.skipUnless(HAVE_DOCSTRINGS, # "test requires docstrings") # #======================================================================= # # doctest driver. # def run_doctest(module, verbosity=None): # """Run doctest on the given module. Return (#failures, #tests). # If optional argument verbosity is not specified (or is None), pass # test_support's belief about verbosity on to doctest. Else doctest's # usual behavior is used (it searches sys.argv for -v). # """ # import doctest # if verbosity is None: # verbosity = verbose # else: # verbosity = None # # Direct doctest output (normally just errors) to real stdout; doctest # # output shouldn't be compared by regrtest. # save_stdout = sys.stdout # sys.stdout = get_original_stdout() # try: # f, t = doctest.testmod(module, verbose=verbosity) # if f: # raise TestFailed("%d of %d doctests failed" % (f, t)) # finally: # sys.stdout = save_stdout # if verbose: # print 'doctest (%s) ... %d tests with zero failures' % (module.__name__, t) # return f, t #======================================================================= # Threading support to prevent reporting refleaks when running regrtest.py -R # NOTE: we use thread._count() rather than threading.enumerate() (or the # moral equivalent thereof) because a threading.Thread object is still alive # until its __bootstrap() method has returned, even after it has been # unregistered from the threading module. # thread._count(), on the other hand, only gets decremented *after* the # __bootstrap() method has returned, which gives us reliable reference counts # at the end of a test run. def threading_setup(): if thread: return (thread._count(),) else: return (1,) def threading_cleanup(nb_threads): if not thread: return _MAX_COUNT = 10 for count in range(_MAX_COUNT): n = thread._count() if n == nb_threads: break time.sleep(0.1) # XXX print a warning in case of failure? # def reap_threads(func): # """Use this function when threads are being used. This will # ensure that the threads are cleaned up even when the test fails. # If threading is unavailable this function does nothing. # """ # if not thread: # return func # @functools.wraps(func) # def decorator(*args): # key = threading_setup() # try: # return func(*args) # finally: # threading_cleanup(*key) # return decorator def reap_children(): """Use this function at the end of test_main() whenever sub-processes are started. This will help ensure that no extra children (zombies) stick around to hog resources and create problems when looking for refleaks. """ # Reap all our dead child processes so we don't leave zombies around. # These hog resources and might be causing some of the buildbots to die. if hasattr(os, 'waitpid'): any_process = -1 while True: try: # This will raise an exception on Windows. That's ok. pid, status = os.waitpid(any_process, os.WNOHANG) if pid == 0: break except: break # @contextlib.contextmanager # def start_threads(threads, unlock=None): # threads = list(threads) # started = [] # try: # try: # for t in threads: # t.start() # started.append(t) # except: # if verbose: # print("Can't start %d threads, only %d threads started" % # (len(threads), len(started))) # raise # yield # finally: # if unlock: # unlock() # endtime = starttime = time.time() # for timeout in range(1, 16): # endtime += 60 # for t in started: # t.join(max(endtime - time.time(), 0.01)) # started = [t for t in started if t.isAlive()] # if not started: # break # if verbose: # print('Unable to join %d threads during a period of ' # '%d minutes' % (len(started), timeout)) # started = [t for t in started if t.isAlive()] # if started: # raise AssertionError('Unable to join %d threads' % len(started)) # @contextlib.contextmanager # def swap_attr(obj, attr, new_val): # """Temporary swap out an attribute with a new object. # Usage: # with swap_attr(obj, "attr", 5): # ... # This will set obj.attr to 5 for the duration of the with: block, # restoring the old value at the end of the block. If `attr` doesn't # exist on `obj`, it will be created and then deleted at the end of the # block. # """ # if hasattr(obj, attr): # real_val = getattr(obj, attr) # setattr(obj, attr, new_val) # try: # yield # finally: # setattr(obj, attr, real_val) # else: # setattr(obj, attr, new_val) # try: # yield # finally: # delattr(obj, attr) # def py3k_bytes(b): # """Emulate the py3k bytes() constructor. # NOTE: This is only a best effort function. # """ # try: # # memoryview? # return b.tobytes() # except AttributeError: # try: # # iterable of ints? # return b"".join(chr(x) for x in b) # except TypeError: # return bytes(b) # def args_from_interpreter_flags(): # """Return a list of command-line arguments reproducing the current # settings in sys.flags.""" # import subprocess # return subprocess._args_from_interpreter_flags() # def strip_python_stderr(stderr): # """Strip the stderr of a Python process from potential debug output # emitted by the interpreter. # This will typically be run on the result of the communicate() method # of a subprocess.Popen object. # """ # stderr = re.sub(br"\[\d+ refs\]\r?\n?$", b"", stderr).strip() # return stderr # def check_free_after_iterating(test, iter, cls, args=()): # class A(cls): # def __del__(self): # done[0] = True # try: # next(it) # except StopIteration: # pass # done = [False] # it = iter(A(*args)) # # Issue 26494: Shouldn't crash # test.assertRaises(StopIteration, next, it) # # The sequence should be deallocated just after the end of iterating # gc_collect() # test.assertTrue(done[0]) ================================================ FILE: third_party/stdlib/test/test_threading.py ================================================ # Very rudimentary test of threading module import test.test_support from test.test_support import verbose, cpython_only #from test.script_helper import assert_python_ok import random import re import sys #thread = test.test_support.import_module('thread') import thread #threading = test.test_support.import_module('threading') import threading import time import unittest import weakref import os #import subprocess #try: # import _testcapi #except ImportError: _testcapi = None from test import lock_tests # A trivial mutable counter. class Counter(object): def __init__(self): self.value = 0 def inc(self): self.value += 1 def dec(self): self.value -= 1 def get(self): return self.value class TestThread(threading.Thread): def __init__(self, name, testcase, sema, mutex, nrunning): threading.Thread.__init__(self, name=name) self.testcase = testcase self.sema = sema self.mutex = mutex self.nrunning = nrunning def run(self): delay = random.random() / 10000.0 if verbose: print 'task %s will run for %s usec' % ( self.name, delay * 1e6) with self.sema: with self.mutex: self.nrunning.inc() if verbose: print self.nrunning.get(), 'tasks are running' self.testcase.assertLessEqual(self.nrunning.get(), 3) time.sleep(delay) if verbose: print 'task', self.name, 'done' with self.mutex: self.nrunning.dec() self.testcase.assertGreaterEqual(self.nrunning.get(), 0) if verbose: print '%s is finished. %d tasks are running' % ( self.name, self.nrunning.get()) class BaseTestCase(unittest.TestCase): def setUp(self): self._threads = test.test_support.threading_setup() def tearDown(self): test.test_support.threading_cleanup(*self._threads) test.test_support.reap_children() class ThreadTests(BaseTestCase): # Create a bunch of threads, let each do some work, wait until all are # done. def test_various_ops(self): # This takes about n/3 seconds to run (about n/3 clumps of tasks, # times about 1 second per clump). NUMTASKS = 10 # no more than 3 of the 10 can run at once sema = threading.BoundedSemaphore(value=3) mutex = threading.RLock() numrunning = Counter() threads = [] for i in range(NUMTASKS): t = TestThread(""%i, self, sema, mutex, numrunning) threads.append(t) self.assertIsNone(t.ident) self.assertRegexpMatches(repr(t), r'^$') t.start() if verbose: print 'waiting for all tasks to complete' for t in threads: t.join(NUMTASKS) self.assertFalse(t.is_alive()) self.assertNotEqual(t.ident, 0) self.assertIsNotNone(t.ident) self.assertRegexpMatches(repr(t), r'^$') if verbose: print 'all tasks done' self.assertEqual(numrunning.get(), 0) def test_ident_of_no_threading_threads(self): # The ident still must work for the main thread and dummy threads. self.assertIsNotNone(threading.currentThread().ident) def f(): ident.append(threading.currentThread().ident) done.set() done = threading.Event() ident = [] thread.start_new_thread(f, ()) done.wait() self.assertIsNotNone(ident[0]) # Kill the "immortal" _DummyThread del threading._active[ident[0]] # run with a small(ish) thread stack size (256kB) def test_various_ops_small_stack(self): if verbose: print 'with 256kB thread stack size...' try: threading.stack_size(262144) except thread.error: self.skipTest('platform does not support changing thread stack size') self.test_various_ops() threading.stack_size(0) # run with a large thread stack size (1MB) def test_various_ops_large_stack(self): if verbose: print 'with 1MB thread stack size...' try: threading.stack_size(0x100000) except thread.error: self.skipTest('platform does not support changing thread stack size') self.test_various_ops() threading.stack_size(0) def test_foreign_thread(self): # Check that a "foreign" thread can use the threading module. def f(mutex): # Calling current_thread() forces an entry for the foreign # thread to get made in the threading._active map. threading.current_thread() mutex.release() mutex = threading.Lock() mutex.acquire() tid = thread.start_new_thread(f, (mutex,)) # Wait for the thread to finish. mutex.acquire() self.assertIn(tid, threading._active) self.assertIsInstance(threading._active[tid], threading._DummyThread) del threading._active[tid] # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) # exposed at the Python level. This test relies on ctypes to get at it. @unittest.skip('grumpy') def test_PyThreadState_SetAsyncExc(self): try: #import ctypes pass except ImportError: self.skipTest('requires ctypes') set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc class AsyncExc(Exception): pass exception = ctypes.py_object(AsyncExc) # First check it works when setting the exception from the same thread. tid = thread.get_ident() try: result = set_async_exc(ctypes.c_long(tid), exception) # The exception is async, so we might have to keep the VM busy until # it notices. while True: pass except AsyncExc: pass else: # This code is unreachable but it reflects the intent. If we wanted # to be smarter the above loop wouldn't be infinite. self.fail("AsyncExc not raised") try: self.assertEqual(result, 1) # one thread state modified except UnboundLocalError: # The exception was raised too quickly for us to get the result. pass # `worker_started` is set by the thread when it's inside a try/except # block waiting to catch the asynchronously set AsyncExc exception. # `worker_saw_exception` is set by the thread upon catching that # exception. worker_started = threading.Event() worker_saw_exception = threading.Event() class Worker(threading.Thread): def run(self): self.id = thread.get_ident() self.finished = False try: while True: worker_started.set() time.sleep(0.1) except AsyncExc: self.finished = True worker_saw_exception.set() t = Worker() t.daemon = True # so if this fails, we don't hang Python at shutdown t.start() if verbose: print " started worker thread" # Try a thread id that doesn't make sense. if verbose: print " trying nonsensical thread id" result = set_async_exc(ctypes.c_long(-1), exception) self.assertEqual(result, 0) # no thread states modified # Now raise an exception in the worker thread. if verbose: print " waiting for worker thread to get started" ret = worker_started.wait() self.assertTrue(ret) if verbose: print " verifying worker hasn't exited" self.assertFalse(t.finished) if verbose: print " attempting to raise asynch exception in worker" result = set_async_exc(ctypes.c_long(t.id), exception) self.assertEqual(result, 1) # one thread state modified if verbose: print " waiting for worker to say it caught the exception" worker_saw_exception.wait(timeout=10) self.assertTrue(t.finished) if verbose: print " all OK -- joining worker" if t.finished: t.join() # else the thread is still running, and we have no way to kill it def test_limbo_cleanup(self): # Issue 7481: Failure to start thread should cleanup the limbo map. def fail_new_thread(*args): raise thread.error() _start_new_thread = threading._start_new_thread threading._start_new_thread = fail_new_thread try: t = threading.Thread(target=lambda: None) self.assertRaises(thread.error, t.start) self.assertFalse( t in threading._limbo, "Failed to cleanup _limbo map on failure of Thread.start().") finally: threading._start_new_thread = _start_new_thread @unittest.skip('grumpy') def test_finalize_runnning_thread(self): # Issue 1402: the PyGILState_Ensure / _Release functions may be called # very late on python exit: on deallocation of a running thread for # example. try: #import ctypes pass except ImportError: self.skipTest('requires ctypes') rc = subprocess.call([sys.executable, "-c", """if 1: import ctypes, sys, time, thread # This lock is used as a simple event variable. ready = thread.allocate_lock() ready.acquire() # Module globals are cleared before __del__ is run # So we save the functions in class dict class C: ensure = ctypes.pythonapi.PyGILState_Ensure release = ctypes.pythonapi.PyGILState_Release def __del__(self): state = self.ensure() self.release(state) def waitingThread(): x = C() ready.release() time.sleep(100) thread.start_new_thread(waitingThread, ()) ready.acquire() # Be sure the other thread is waiting. sys.exit(42) """]) self.assertEqual(rc, 42) @unittest.skip('grumpy') def test_finalize_with_trace(self): # Issue1733757 # Avoid a deadlock when sys.settrace steps into threading._shutdown p = subprocess.Popen([sys.executable, "-c", """if 1: import sys, threading # A deadlock-killer, to prevent the # testsuite to hang forever def killer(): import os, time time.sleep(2) print 'program blocked; aborting' os._exit(2) t = threading.Thread(target=killer) t.daemon = True t.start() # This is the trace function def func(frame, event, arg): threading.current_thread() return func sys.settrace(func) """], stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.addCleanup(p.stdout.close) self.addCleanup(p.stderr.close) stdout, stderr = p.communicate() rc = p.returncode self.assertFalse(rc == 2, "interpreted was blocked") self.assertTrue(rc == 0, "Unexpected error: " + repr(stderr)) @unittest.skip('grumpy') def test_join_nondaemon_on_shutdown(self): # Issue 1722344 # Raising SystemExit skipped threading._shutdown p = subprocess.Popen([sys.executable, "-c", """if 1: import threading from time import sleep def child(): sleep(1) # As a non-daemon thread we SHOULD wake up and nothing # should be torn down yet print "Woke up, sleep function is:", sleep threading.Thread(target=child).start() raise SystemExit """], stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.addCleanup(p.stdout.close) self.addCleanup(p.stderr.close) stdout, stderr = p.communicate() self.assertEqual(stdout.strip(), "Woke up, sleep function is: ") stderr = re.sub(r"^\[\d+ refs\]", "", stderr, re.MULTILINE).strip() self.assertEqual(stderr, "") @unittest.skip('grumpy') def test_enumerate_after_join(self): # Try hard to trigger #1703448: a thread is still returned in # threading.enumerate() after it has been join()ed. enum = threading.enumerate old_interval = sys.getcheckinterval() try: for i in xrange(1, 100): # Try a couple times at each thread-switching interval # to get more interleavings. sys.setcheckinterval(i // 5) t = threading.Thread(target=lambda: None) t.start() t.join() l = enum() self.assertNotIn(t, l, "#1703448 triggered after %d trials: %s" % (i, l)) finally: sys.setcheckinterval(old_interval) @unittest.skip('grumpy') def test_no_refcycle_through_target(self): class RunSelfFunction(object): def __init__(self, should_raise): # The links in this refcycle from Thread back to self # should be cleaned up when the thread completes. self.should_raise = should_raise self.thread = threading.Thread(target=self._run, args=(self,), kwargs={'yet_another':self}) self.thread.start() def _run(self, other_ref, yet_another): if self.should_raise: raise SystemExit cyclic_object = RunSelfFunction(should_raise=False) weak_cyclic_object = weakref.ref(cyclic_object) cyclic_object.thread.join() del cyclic_object self.assertEqual(None, weak_cyclic_object(), msg=('%d references still around' % sys.getrefcount(weak_cyclic_object()))) raising_cyclic_object = RunSelfFunction(should_raise=True) weak_raising_cyclic_object = weakref.ref(raising_cyclic_object) raising_cyclic_object.thread.join() del raising_cyclic_object self.assertEqual(None, weak_raising_cyclic_object(), msg=('%d references still around' % sys.getrefcount(weak_raising_cyclic_object()))) @unittest.skip('grumpy') @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. code = """if 1: import thread, threading, os, time def background_thread(evt): # Creates and registers the _DummyThread instance threading.current_thread() evt.set() time.sleep(10) evt = threading.Event() thread.start_new_thread(background_thread, (evt,)) evt.wait() assert threading.active_count() == 2, threading.active_count() if os.fork() == 0: assert threading.active_count() == 1, threading.active_count() os._exit(0) else: os.wait() """ _, out, err = assert_python_ok("-c", code) self.assertEqual(out, '') self.assertEqual(err, '') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_is_alive_after_fork(self): # Try hard to trigger #18418: is_alive() could sometimes be True on # threads that vanished after a fork. old_interval = sys.getcheckinterval() # Make the bug more likely to manifest. sys.setcheckinterval(10) try: for i in range(20): t = threading.Thread(target=lambda: None) t.start() pid = os.fork() if pid == 0: os._exit(1 if t.is_alive() else 0) else: t.join() pid, status = os.waitpid(pid, 0) self.assertEqual(0, status) finally: sys.setcheckinterval(old_interval) def test_BoundedSemaphore_limit(self): # BoundedSemaphore should raise ValueError if released too often. for limit in range(1, 10): bs = threading.BoundedSemaphore(limit) threads = [threading.Thread(target=bs.acquire) for _ in range(limit)] for t in threads: t.start() for t in threads: t.join() threads = [threading.Thread(target=bs.release) for _ in range(limit)] for t in threads: t.start() for t in threads: t.join() self.assertRaises(ValueError, bs.release) class ThreadJoinOnShutdown(BaseTestCase): # Between fork() and exec(), only async-safe functions are allowed (issues # #12316 and #11870), and fork() from a worker thread is known to trigger # problems with some operating systems (issue #3863): skip problematic tests # on platforms known to behave badly. platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', 'os2emx') def _run_and_join(self, script): script = """if 1: import sys, os, time, threading # a thread, which waits for the main program to terminate def joiningfunc(mainthread): mainthread.join() print 'end of thread' \n""" + script p = subprocess.Popen([sys.executable, "-c", script], stdout=subprocess.PIPE) rc = p.wait() data = p.stdout.read().replace('\r', '') p.stdout.close() self.assertEqual(data, "end of main\nend of thread\n") self.assertFalse(rc == 2, "interpreter was blocked") self.assertTrue(rc == 0, "Unexpected error") @unittest.skip('grumpy') def test_1_join_on_shutdown(self): # The usual case: on exit, wait for a non-daemon thread script = """if 1: import os t = threading.Thread(target=joiningfunc, args=(threading.current_thread(),)) t.start() time.sleep(0.1) print 'end of main' """ self._run_and_join(script) @unittest.skip('grumpy') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_2_join_in_forked_process(self): # Like the test above, but from a forked interpreter script = """if 1: childpid = os.fork() if childpid != 0: os.waitpid(childpid, 0) sys.exit(0) t = threading.Thread(target=joiningfunc, args=(threading.current_thread(),)) t.start() print 'end of main' """ self._run_and_join(script) @unittest.skip('grumpy') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. script = """if 1: main_thread = threading.current_thread() def worker(): childpid = os.fork() if childpid != 0: os.waitpid(childpid, 0) sys.exit(0) t = threading.Thread(target=joiningfunc, args=(main_thread,)) print 'end of main' t.start() t.join() # Should not block: main_thread is already stopped w = threading.Thread(target=worker) w.start() """ self._run_and_join(script) def assertScriptHasOutput(self, script, expected_output): p = subprocess.Popen([sys.executable, "-c", script], stdout=subprocess.PIPE) rc = p.wait() data = p.stdout.read().decode().replace('\r', '') self.assertEqual(rc, 0, "Unexpected error") self.assertEqual(data, expected_output) @unittest.skip('grumpy') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_4_joining_across_fork_in_worker_thread(self): # There used to be a possible deadlock when forking from a child # thread. See http://bugs.python.org/issue6643. # The script takes the following steps: # - The main thread in the parent process starts a new thread and then # tries to join it. # - The join operation acquires the Lock inside the thread's _block # Condition. (See threading.py:Thread.join().) # - We stub out the acquire method on the condition to force it to wait # until the child thread forks. (See LOCK ACQUIRED HERE) # - The child thread forks. (See LOCK HELD and WORKER THREAD FORKS # HERE) # - The main thread of the parent process enters Condition.wait(), # which releases the lock on the child thread. # - The child process returns. Without the necessary fix, when the # main thread of the child process (which used to be the child thread # in the parent process) attempts to exit, it will try to acquire the # lock in the Thread._block Condition object and hang, because the # lock was held across the fork. script = """if 1: import os, time, threading finish_join = False start_fork = False def worker(): # Wait until this thread's lock is acquired before forking to # create the deadlock. global finish_join while not start_fork: time.sleep(0.01) # LOCK HELD: Main thread holds lock across this call. childpid = os.fork() finish_join = True if childpid != 0: # Parent process just waits for child. os.waitpid(childpid, 0) # Child process should just return. w = threading.Thread(target=worker) # Stub out the private condition variable's lock acquire method. # This acquires the lock and then waits until the child has forked # before returning, which will release the lock soon after. If # someone else tries to fix this test case by acquiring this lock # before forking instead of resetting it, the test case will # deadlock when it shouldn't. condition = w._block orig_acquire = condition.acquire call_count_lock = threading.Lock() call_count = 0 def my_acquire(): global call_count global start_fork orig_acquire() # LOCK ACQUIRED HERE start_fork = True if call_count == 0: while not finish_join: time.sleep(0.01) # WORKER THREAD FORKS HERE with call_count_lock: call_count += 1 condition.acquire = my_acquire w.start() w.join() print('end of main') """ self.assertScriptHasOutput(script, "end of main\n") @unittest.skip('grumpy') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_5_clear_waiter_locks_to_avoid_crash(self): # Check that a spawned thread that forks doesn't segfault on certain # platforms, namely OS X. This used to happen if there was a waiter # lock in the thread's condition variable's waiters list. Even though # we know the lock will be held across the fork, it is not safe to # release locks held across forks on all platforms, so releasing the # waiter lock caused a segfault on OS X. Furthermore, since locks on # OS X are (as of this writing) implemented with a mutex + condition # variable instead of a semaphore, while we know that the Python-level # lock will be acquired, we can't know if the internal mutex will be # acquired at the time of the fork. script = """if True: import os, time, threading start_fork = False def worker(): # Wait until the main thread has attempted to join this thread # before continuing. while not start_fork: time.sleep(0.01) childpid = os.fork() if childpid != 0: # Parent process just waits for child. (cpid, rc) = os.waitpid(childpid, 0) assert cpid == childpid assert rc == 0 print('end of worker thread') else: # Child process should just return. pass w = threading.Thread(target=worker) # Stub out the private condition variable's _release_save method. # This releases the condition's lock and flips the global that # causes the worker to fork. At this point, the problematic waiter # lock has been acquired once by the waiter and has been put onto # the waiters list. condition = w._block orig_release_save = condition._release_save def my_release_save(): global start_fork orig_release_save() # Waiter lock held here, condition lock released. start_fork = True condition._release_save = my_release_save w.start() w.join() print('end of main thread') """ output = "end of worker thread\nend of main thread\n" self.assertScriptHasOutput(script, output) @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with # the ad-hoc TLS implementation. def do_fork_and_wait(): # just fork a child process and wait it pid = os.fork() if pid > 0: os.waitpid(pid, 0) else: os._exit(0) # start a bunch of threads that will fork() child processes threads = [] for i in range(16): t = threading.Thread(target=do_fork_and_wait) threads.append(t) t.start() for t in threads: t.join() @cpython_only @unittest.skipIf(_testcapi is None, "need _testcapi module") def test_frame_tstate_tracing(self): # Issue #14432: Crash when a generator is created in a C thread that is # destroyed while the generator is still used. The issue was that a # generator contains a frame, and the frame kept a reference to the # Python state of the destroyed C thread. The crash occurs when a trace # function is setup. def noop_trace(frame, event, arg): # no operation return noop_trace def generator(): while 1: yield "generator" def callback(): if callback.gen is None: callback.gen = generator() return next(callback.gen) callback.gen = None old_trace = sys.gettrace() sys.settrace(noop_trace) try: # Install a trace function threading.settrace(noop_trace) # Create a generator in a C thread which exits after the call _testcapi.call_in_temporary_c_thread(callback) # Call the generator in a different Python thread, check that the # generator didn't keep a reference to the destroyed thread state for test in range(3): # The trace function is still called here callback() finally: sys.settrace(old_trace) class ThreadingExceptionTests(BaseTestCase): # A RuntimeError should be raised if Thread.start() is called # multiple times. def test_start_thread_again(self): thread = threading.Thread() thread.start() self.assertRaises(RuntimeError, thread.start) def test_joining_current_thread(self): current_thread = threading.current_thread() self.assertRaises(RuntimeError, current_thread.join); def test_joining_inactive_thread(self): thread = threading.Thread() self.assertRaises(RuntimeError, thread.join) def test_daemonize_active_thread(self): thread = threading.Thread() thread.start() self.assertRaises(RuntimeError, setattr, thread, "daemon", True) @unittest.skip('grumpy') def test_print_exception(self): script = r"""if 1: import threading import time running = False def run(): global running running = True while running: time.sleep(0.01) 1.0/0.0 t = threading.Thread(target=run) t.start() while not running: time.sleep(0.01) running = False t.join() """ rc, out, err = assert_python_ok("-c", script) self.assertEqual(out, '') self.assertIn("Exception in thread", err) self.assertIn("Traceback (most recent call last):", err) self.assertIn("ZeroDivisionError", err) self.assertNotIn("Unhandled exception", err) @unittest.skip('grumpy') def test_print_exception_stderr_is_none_1(self): script = r"""if 1: import sys import threading import time running = False def run(): global running running = True while running: time.sleep(0.01) 1.0/0.0 t = threading.Thread(target=run) t.start() while not running: time.sleep(0.01) sys.stderr = None running = False t.join() """ rc, out, err = assert_python_ok("-c", script) self.assertEqual(out, '') self.assertIn("Exception in thread", err) self.assertIn("Traceback (most recent call last):", err) self.assertIn("ZeroDivisionError", err) self.assertNotIn("Unhandled exception", err) @unittest.skip('grumpy') def test_print_exception_stderr_is_none_2(self): script = r"""if 1: import sys import threading import time running = False def run(): global running running = True while running: time.sleep(0.01) 1.0/0.0 sys.stderr = None t = threading.Thread(target=run) t.start() while not running: time.sleep(0.01) running = False t.join() """ rc, out, err = assert_python_ok("-c", script) self.assertEqual(out, '') self.assertNotIn("Unhandled exception", err) class LockTests(lock_tests.LockTests): locktype = staticmethod(threading.Lock) class RLockTests(lock_tests.RLockTests): locktype = staticmethod(threading.RLock) class EventTests(lock_tests.EventTests): eventtype = staticmethod(threading.Event) class ConditionAsRLockTests(lock_tests.RLockTests): # Condition uses an RLock by default and exports its API. locktype = staticmethod(threading.Condition) class ConditionTests(lock_tests.ConditionTests): condtype = staticmethod(threading.Condition) class SemaphoreTests(lock_tests.SemaphoreTests): semtype = staticmethod(threading.Semaphore) class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): semtype = staticmethod(threading.BoundedSemaphore) @unittest.skip('grumpy') @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem') def test_recursion_limit(self): # Issue 9670 # test that excessive recursion within a non-main thread causes # an exception rather than crashing the interpreter on platforms # like Mac OS X or FreeBSD which have small default stack sizes # for threads script = """if True: import threading def recurse(): return recurse() def outer(): try: recurse() except RuntimeError: pass w = threading.Thread(target=outer) w.start() w.join() print('end of main thread') """ expected_output = "end of main thread\n" p = subprocess.Popen([sys.executable, "-c", script], stdout=subprocess.PIPE) stdout, stderr = p.communicate() data = stdout.decode().replace('\r', '') self.assertEqual(p.returncode, 0, "Unexpected error") self.assertEqual(data, expected_output) def test_main(): test.test_support.run_unittest(LockTests, RLockTests, EventTests, ConditionAsRLockTests, ConditionTests, SemaphoreTests, BoundedSemaphoreTests, ThreadTests, ThreadJoinOnShutdown, ThreadingExceptionTests, ) if __name__ == "__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_tuple.py ================================================ from test import test_support, seq_tests # import gc class TupleTest(seq_tests.CommonTest): type2test = tuple def test_constructors(self): super(TupleTest, self).test_constructors() # calling built-in types without argument must return empty self.assertEqual(tuple(), ()) t0_3 = (0, 1, 2, 3) t0_3_bis = tuple(t0_3) self.assertTrue(t0_3 is t0_3_bis) self.assertEqual(tuple([]), ()) self.assertEqual(tuple([0, 1, 2, 3]), (0, 1, 2, 3)) self.assertEqual(tuple(''), ()) self.assertEqual(tuple('spam'), ('s', 'p', 'a', 'm')) def test_truth(self): super(TupleTest, self).test_truth() self.assertTrue(not ()) self.assertTrue((42, )) def test_len(self): super(TupleTest, self).test_len() self.assertEqual(len(()), 0) self.assertEqual(len((0,)), 1) self.assertEqual(len((0, 1, 2)), 3) def test_iadd(self): super(TupleTest, self).test_iadd() u = (0, 1) u2 = u u += (2, 3) self.assertTrue(u is not u2) def test_imul(self): super(TupleTest, self).test_imul() u = (0, 1) u2 = u u *= 3 self.assertTrue(u is not u2) def test_tupleresizebug(self): # Check that a specific bug in _PyTuple_Resize() is squashed. def f(): for i in range(1000): yield i self.assertEqual(list(tuple(f())), range(1000)) def test_hash(self): # See SF bug 942952: Weakness in tuple hash # The hash should: # be non-commutative # should spread-out closely spaced values # should not exhibit cancellation in tuples like (x,(x,y)) # should be distinct from element hashes: hash(x)!=hash((x,)) # This test exercises those cases. # For a pure random hash and N=50, the expected number of occupied # buckets when tossing 252,600 balls into 2**32 buckets # is 252,592.6, or about 7.4 expected collisions. The # standard deviation is 2.73. On a box with 64-bit hash # codes, no collisions are expected. Here we accept no # more than 15 collisions. Any worse and the hash function # is sorely suspect. N=50 base = range(N) xp = [(i, j) for i in base for j in base] inps = base + [(i, j) for i in base for j in xp] + \ [(i, j) for i in xp for j in base] + xp + zip(base) collisions = len(inps) - len(set(map(hash, inps))) self.assertTrue(collisions <= 15) def test_repr(self): l0 = tuple() l2 = (0, 1, 2) a0 = self.type2test(l0) a2 = self.type2test(l2) self.assertEqual(str(a0), repr(l0)) self.assertEqual(str(a2), repr(l2)) self.assertEqual(repr(a0), "()") self.assertEqual(repr(a2), "(0, 1, 2)") # def _not_tracked(self, t): # # Nested tuples can take several collections to untrack # gc.collect() # gc.collect() # self.assertFalse(gc.is_tracked(t), t) # def _tracked(self, t): # self.assertTrue(gc.is_tracked(t), t) # gc.collect() # gc.collect() # self.assertTrue(gc.is_tracked(t), t) # @test_support.cpython_only # def test_track_literals(self): # # Test GC-optimization of tuple literals # x, y, z = 1.5, "a", [] # self._not_tracked(()) # self._not_tracked((1,)) # self._not_tracked((1, 2)) # self._not_tracked((1, 2, "a")) # self._not_tracked((1, 2, (None, True, False, ()), int)) # self._not_tracked((object(),)) # self._not_tracked(((1, x), y, (2, 3))) # # Tuples with mutable elements are always tracked, even if those # # elements are not tracked right now. # self._tracked(([],)) # self._tracked(([1],)) # self._tracked(({},)) # self._tracked((set(),)) # self._tracked((x, y, z)) # def check_track_dynamic(self, tp, always_track): # x, y, z = 1.5, "a", [] # check = self._tracked if always_track else self._not_tracked # check(tp()) # check(tp([])) # check(tp(set())) # check(tp([1, x, y])) # check(tp(obj for obj in [1, x, y])) # check(tp(set([1, x, y]))) # check(tp(tuple([obj]) for obj in [1, x, y])) # check(tuple(tp([obj]) for obj in [1, x, y])) # self._tracked(tp([z])) # self._tracked(tp([[x, y]])) # self._tracked(tp([{x: y}])) # self._tracked(tp(obj for obj in [x, y, z])) # self._tracked(tp(tuple([obj]) for obj in [x, y, z])) # self._tracked(tuple(tp([obj]) for obj in [x, y, z])) # @test_support.cpython_only # def test_track_dynamic(self): # # Test GC-optimization of dynamically constructed tuples. # self.check_track_dynamic(tuple, False) # @test_support.cpython_only # def test_track_subtypes(self): # # Tuple subtypes must always be tracked # class MyTuple(tuple): # pass # self.check_track_dynamic(MyTuple, True) # @test_support.cpython_only # def test_bug7466(self): # # Trying to untrack an unfinished tuple could crash Python # self._not_tracked(tuple(gc.collect() for i in range(101))) def test_main(): test_support.run_unittest(TupleTest) if __name__=="__main__": test_main() ================================================ FILE: third_party/stdlib/test/test_uu.py ================================================ """ Tests for uu module. Nick Mathewson """ import unittest from test import test_support import sys, os, uu, cStringIO import uu plaintext = "The smooth-scaled python crept over the sleeping dog\n" encodedtext = """\ M5&AE('-M;V]T:\"US8V%L960@<'ET:&]N(&-R97!T(&]V97(@=&AE('-L965P (:6YG(&1O9PH """ encodedtextwrapped = "begin %03o %s\n" + encodedtext.replace("%", "%%") + "\n \nend\n" class UUTest(unittest.TestCase): def test_encode(self): inp = cStringIO.StringIO(plaintext) out = cStringIO.StringIO() uu.encode(inp, out, "t1") self.assertEqual(out.getvalue(), encodedtextwrapped % (0666, "t1")) inp = cStringIO.StringIO(plaintext) out = cStringIO.StringIO() uu.encode(inp, out, "t1", 0644) self.assertEqual(out.getvalue(), encodedtextwrapped % (0644, "t1")) def test_decode(self): inp = cStringIO.StringIO(encodedtextwrapped % (0666, "t1")) out = cStringIO.StringIO() uu.decode(inp, out) self.assertEqual(out.getvalue(), plaintext) inp = cStringIO.StringIO( "UUencoded files may contain many lines,\n" + "even some that have 'begin' in them.\n" + encodedtextwrapped % (0666, "t1") ) out = cStringIO.StringIO() uu.decode(inp, out) self.assertEqual(out.getvalue(), plaintext) def test_truncatedinput(self): inp = cStringIO.StringIO("begin 644 t1\n" + encodedtext) out = cStringIO.StringIO() try: uu.decode(inp, out) self.fail("No exception raised") except uu.Error, e: self.assertEqual(str(e), "Truncated input file") def test_missingbegin(self): inp = cStringIO.StringIO("") out = cStringIO.StringIO() try: uu.decode(inp, out) self.fail("No exception raised") except uu.Error, e: self.assertEqual(str(e), "No valid begin line found in input file") def test_garbage_padding(self): # Issue #22406 encodedtext = ( "begin 644 file\n" # length 1; bits 001100 111111 111111 111111 "\x21\x2C\x5F\x5F\x5F\n" "\x20\n" "end\n" ) plaintext = "\x33" # 00110011 inp = cStringIO.StringIO(encodedtext) out = cStringIO.StringIO() uu.decode(inp, out, quiet=True) self.assertEqual(out.getvalue(), plaintext) #import codecs #decoded = codecs.decode(encodedtext, "uu_codec") #self.assertEqual(decoded, plaintext) class UUStdIOTest(unittest.TestCase): def setUp(self): self.stdin = sys.stdin self.stdout = sys.stdout def tearDown(self): sys.stdin = self.stdin sys.stdout = self.stdout def test_encode(self): sys.stdin = cStringIO.StringIO(plaintext) sys.stdout = cStringIO.StringIO() uu.encode("-", "-", "t1", 0666) self.assertEqual( sys.stdout.getvalue(), encodedtextwrapped % (0666, "t1") ) def test_decode(self): sys.stdin = cStringIO.StringIO(encodedtextwrapped % (0666, "t1")) sys.stdout = cStringIO.StringIO() uu.decode("-", "-") self.assertEqual(sys.stdout.getvalue(), plaintext) class UUFileTest(unittest.TestCase): def _kill(self, f): # close and remove file try: f.close() except (SystemExit, KeyboardInterrupt): raise except: pass try: os.unlink(f.name) except (SystemExit, KeyboardInterrupt): raise # except: # pass def setUp(self): self.tmpin = test_support.TESTFN + "i" self.tmpout = test_support.TESTFN + "o" def tearDown(self): del self.tmpin del self.tmpout def test_encode(self): fin = fout = None try: test_support.unlink(self.tmpin) fin = open(self.tmpin, 'wb') fin.write(plaintext) fin.close() fin = open(self.tmpin, 'rb') fout = open(self.tmpout, 'w') uu.encode(fin, fout, self.tmpin, mode=0644) fin.close() fout.close() fout = open(self.tmpout, 'r') s = fout.read() fout.close() self.assertEqual(s, encodedtextwrapped % (0644, self.tmpin)) # in_file and out_file as filenames uu.encode(self.tmpin, self.tmpout, self.tmpin, mode=0644) fout = open(self.tmpout, 'r') s = fout.read() fout.close() self.assertEqual(s, encodedtextwrapped % (0644, self.tmpin)) finally: self._kill(fin) self._kill(fout) def test_decode(self): f = None try: test_support.unlink(self.tmpin) f = open(self.tmpin, 'w') f.write(encodedtextwrapped % (0644, self.tmpout)) f.close() f = open(self.tmpin, 'r') uu.decode(f) f.close() f = open(self.tmpout, 'r') s = f.read() f.close() self.assertEqual(s, plaintext) # XXX is there an xp way to verify the mode? finally: self._kill(f) def test_decode_filename(self): f = None try: test_support.unlink(self.tmpin) f = open(self.tmpin, 'w') f.write(encodedtextwrapped % (0644, self.tmpout)) f.close() uu.decode(self.tmpin) f = open(self.tmpout, 'r') s = f.read() f.close() self.assertEqual(s, plaintext) finally: self._kill(f) def test_decodetwice(self): # Verify that decode() will refuse to overwrite an existing file f = None try: f = cStringIO.StringIO(encodedtextwrapped % (0644, self.tmpout)) f = open(self.tmpin, 'r') uu.decode(f) f.close() f = open(self.tmpin, 'r') self.assertRaises(uu.Error, uu.decode, f) f.close() finally: self._kill(f) def test_main(): test_support.run_unittest(UUTest, UUStdIOTest, UUFileTest) if __name__=="__main__": test_main() ================================================ FILE: third_party/stdlib/textwrap.py ================================================ """Text wrapping and filling. """ # Copyright (C) 1999-2001 Gregory P. Ward. # Copyright (C) 2002, 2003 Python Software Foundation. # Written by Greg Ward __revision__ = "$Id$" import string, re try: _unicode = unicode except NameError: # If Python is built without Unicode support, the unicode type # will not exist. Fake one. class _unicode(object): pass # Do the right thing with boolean values for all known Python versions # (so this module can be copied to projects that don't depend on Python # 2.3, e.g. Optik and Docutils) by uncommenting the block of code below. #try: # True, False #except NameError: # (True, False) = (1, 0) __all__ = ['TextWrapper', 'wrap', 'fill', 'dedent'] # Hardcode the recognized whitespace characters to the US-ASCII # whitespace characters. The main reason for doing this is that in # ISO-8859-1, 0xa0 is non-breaking whitespace, so in certain locales # that character winds up in string.whitespace. Respecting # string.whitespace in those cases would 1) make textwrap treat 0xa0 the # same as any other whitespace char, which is clearly wrong (it's a # *non-breaking* space), 2) possibly cause problems with Unicode, # since 0xa0 is not in range(128). _whitespace = '\t\n\x0b\x0c\r ' class TextWrapper(object): """ Object for wrapping/filling text. The public interface consists of the wrap() and fill() methods; the other methods are just there for subclasses to override in order to tweak the default behaviour. If you want to completely replace the main wrapping algorithm, you'll probably have to override _wrap_chunks(). Several instance attributes control various aspects of wrapping: width (default: 70) the maximum width of wrapped lines (unless break_long_words is false) initial_indent (default: "") string that will be prepended to the first line of wrapped output. Counts towards the line's width. subsequent_indent (default: "") string that will be prepended to all lines save the first of wrapped output; also counts towards each line's width. expand_tabs (default: true) Expand tabs in input text to spaces before further processing. Each tab will become 1 .. 8 spaces, depending on its position in its line. If false, each tab is treated as a single character. replace_whitespace (default: true) Replace all whitespace characters in the input text by spaces after tab expansion. Note that if expand_tabs is false and replace_whitespace is true, every tab will be converted to a single space! fix_sentence_endings (default: false) Ensure that sentence-ending punctuation is always followed by two spaces. Off by default because the algorithm is (unavoidably) imperfect. break_long_words (default: true) Break words longer than 'width'. If false, those words will not be broken, and some lines might be longer than 'width'. break_on_hyphens (default: true) Allow breaking hyphenated words. If true, wrapping will occur preferably on whitespaces and right after hyphens part of compound words. drop_whitespace (default: true) Drop leading and trailing whitespace from lines. """ # whitespace_trans = string.maketrans(_whitespace, ' ' * len(_whitespace)) whitespace_trans = '\x00\x01\x02\x03\x04\x05\x06\x07\x08 \x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff' unicode_whitespace_trans = {} uspace = ord(u' ') for x in map(ord, _whitespace): unicode_whitespace_trans[x] = uspace # This funky little regex is just the trick for splitting # text up into word-wrappable chunks. E.g. # "Hello there -- you goof-ball, use the -b option!" # splits into # Hello/ /there/ /--/ /you/ /goof-/ball,/ /use/ /the/ /-b/ /option! # (after stripping out empty strings). wordsep_re = re.compile( r'(\s+|' # any whitespace r'[^\s\w]*\w+[^0-9\W]-(?=\w+[^0-9\W])|' # hyphenated words r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))') # em-dash # This less funky little regex just split on recognized spaces. E.g. # "Hello there -- you goof-ball, use the -b option!" # splits into # Hello/ /there/ /--/ /you/ /goof-ball,/ /use/ /the/ /-b/ /option!/ wordsep_simple_re = re.compile(r'(\s+)') # XXX this is not locale- or charset-aware -- string.lowercase # is US-ASCII only (and therefore English-only) sentence_end_re = re.compile(r'[%s]' # lowercase letter r'[\.\!\?]' # sentence-ending punct. r'[\"\']?' # optional end-of-quote r'\Z' # end of chunk % string.lowercase) def __init__(self, width=70, initial_indent="", subsequent_indent="", expand_tabs=True, replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, drop_whitespace=True, break_on_hyphens=True): self.width = width self.initial_indent = initial_indent self.subsequent_indent = subsequent_indent self.expand_tabs = expand_tabs self.replace_whitespace = replace_whitespace self.fix_sentence_endings = fix_sentence_endings self.break_long_words = break_long_words self.drop_whitespace = drop_whitespace self.break_on_hyphens = break_on_hyphens # recompile the regexes for Unicode mode -- done in this clumsy way for # backwards compatibility because it's rather common to monkey-patch # the TextWrapper class' wordsep_re attribute. self.wordsep_re_uni = re.compile(self.wordsep_re.pattern, re.U) self.wordsep_simple_re_uni = re.compile( self.wordsep_simple_re.pattern, re.U) # -- Private methods ----------------------------------------------- # (possibly useful for subclasses to override) def _munge_whitespace(self, text): """_munge_whitespace(text : string) -> string Munge whitespace in text: expand tabs and convert all other whitespace characters to spaces. Eg. " foo\\tbar\\n\\nbaz" becomes " foo bar baz". """ if self.expand_tabs: # text = text.expandtabs() text = ' '.join((' '.join(text.split('\n'))).split('\t')) if self.replace_whitespace: # if isinstance(text, str): # text = text.translate(self.whitespace_trans) # elif isinstance(text, _unicode): # text = text.translate(self.unicode_whitespace_trans) text = ' '.join(' '.join(text.split('\n')).split('\t')) return text def _split(self, text): """_split(text : string) -> [string] Split the text to wrap into indivisible chunks. Chunks are not quite the same as words; see _wrap_chunks() for full details. As an example, the text Look, goof-ball -- use the -b option! breaks into the following chunks: 'Look,', ' ', 'goof-', 'ball', ' ', '--', ' ', 'use', ' ', 'the', ' ', '-b', ' ', 'option!' if break_on_hyphens is True, or in: 'Look,', ' ', 'goof-ball', ' ', '--', ' ', 'use', ' ', 'the', ' ', '-b', ' ', option!' otherwise. """ if isinstance(text, _unicode): if self.break_on_hyphens: pat = self.wordsep_re_uni else: pat = self.wordsep_simple_re_uni else: if self.break_on_hyphens: pat = self.wordsep_re else: pat = self.wordsep_simple_re chunks = pat.split(text) # chunks = filter(None, chunks) # remove empty chunks chunks = [x for x in chunks if x is not None] return chunks def _fix_sentence_endings(self, chunks): """_fix_sentence_endings(chunks : [string]) Correct for sentence endings buried in 'chunks'. Eg. when the original text contains "... foo.\\nBar ...", munge_whitespace() and split() will convert that to [..., "foo.", " ", "Bar", ...] which has one too few spaces; this method simply changes the one space to two. """ i = 0 patsearch = self.sentence_end_re.search while i < len(chunks)-1: if chunks[i+1] == " " and patsearch(chunks[i]): chunks[i+1] = " " i += 2 else: i += 1 def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): """_handle_long_word(chunks : [string], cur_line : [string], cur_len : int, width : int) Handle a chunk of text (most likely a word, not whitespace) that is too long to fit in any line. """ # Figure out when indent is larger than the specified width, and make # sure at least one character is stripped off on every pass if width < 1: space_left = 1 else: space_left = width - cur_len # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. if self.break_long_words: cur_line.append(reversed_chunks[-1][:space_left]) reversed_chunks[-1] = reversed_chunks[-1][space_left:] # Otherwise, we have to preserve the long word intact. Only add # it to the current line if there's nothing already there -- # that minimizes how much we violate the width constraint. elif not cur_line: cur_line.append(reversed_chunks.pop()) # If we're not allowed to break long words, and there's already # text on the current line, do nothing. Next time through the # main loop of _wrap_chunks(), we'll wind up here again, but # cur_len will be zero, so the next line will be entirely # devoted to the long word that we can't handle right now. def _wrap_chunks(self, chunks): """_wrap_chunks(chunks : [string]) -> [string] Wrap a sequence of text chunks and return a list of lines of length 'self.width' or less. (If 'break_long_words' is false, some lines may be longer than this.) Chunks correspond roughly to words and the whitespace between them: each chunk is indivisible (modulo 'break_long_words'), but a line break can come between any two chunks. Chunks should not have internal whitespace; ie. a chunk is either all whitespace or a "word". Whitespace chunks will be removed from the beginning and end of lines, but apart from that whitespace is preserved. """ lines = [] if self.width <= 0: raise ValueError("invalid width %r (must be > 0)" % self.width) # Arrange in reverse order so items can be efficiently popped # from a stack of chucks. chunks.reverse() while chunks: # Start the list of chunks that will make up the current line. # cur_len is just the length of all the chunks in cur_line. cur_line = [] cur_len = 0 # Figure out which static string will prefix this line. if lines: indent = self.subsequent_indent else: indent = self.initial_indent # Maximum width for this line. width = self.width - len(indent) # First chunk on line is whitespace -- drop it, unless this # is the very beginning of the text (ie. no lines started yet). if self.drop_whitespace and chunks[-1].strip() == '' and lines: # del chunks[-1] chunks.pop() while chunks: l = len(chunks[-1]) # Can at least squeeze this chunk onto the current line. if cur_len + l <= width: cur_line.append(chunks.pop()) cur_len += l # Nope, this line is full. else: break # The current line is full, and the next chunk is too big to # fit on *any* line (not just this one). if chunks and len(chunks[-1]) > width: self._handle_long_word(chunks, cur_line, cur_len, width) # If the last chunk on this line is all whitespace, drop it. if self.drop_whitespace and cur_line and cur_line[-1].strip() == '': # del cur_line[-1] cur_line.pop() # Convert current line back to a string and store it in list # of all lines (return value). if cur_line: lines.append(indent + ''.join(cur_line)) return lines # -- Public interface ---------------------------------------------- def wrap(self, text): """wrap(text : string) -> [string] Reformat the single paragraph in 'text' so it fits in lines of no more than 'self.width' columns, and return a list of wrapped lines. Tabs in 'text' are expanded with string.expandtabs(), and all other whitespace characters (including newline) are converted to space. """ text = self._munge_whitespace(text) chunks = self._split(text) if self.fix_sentence_endings: self._fix_sentence_endings(chunks) return self._wrap_chunks(chunks) def fill(self, text): """fill(text : string) -> string Reformat the single paragraph in 'text' to fit in lines of no more than 'self.width' columns, and return a new string containing the entire wrapped paragraph. """ return "\n".join(self.wrap(text)) # -- Convenience interface --------------------------------------------- def wrap(text, width=70, **kwargs): """Wrap a single paragraph of text, returning a list of wrapped lines. Reformat the single paragraph in 'text' so it fits in lines of no more than 'width' columns, and return a list of wrapped lines. By default, tabs in 'text' are expanded with string.expandtabs(), and all other whitespace characters (including newline) are converted to space. See TextWrapper class for available keyword args to customize wrapping behaviour. """ w = TextWrapper(width=width, **kwargs) return w.wrap(text) def fill(text, width=70, **kwargs): """Fill a single paragraph of text, returning a new string. Reformat the single paragraph in 'text' to fit in lines of no more than 'width' columns, and return a new string containing the entire wrapped paragraph. As with wrap(), tabs are expanded and other whitespace characters converted to space. See TextWrapper class for available keyword args to customize wrapping behaviour. """ w = TextWrapper(width=width, **kwargs) return w.fill(text) # -- Loosely related functionality ------------------------------------- _whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE) _leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE) def dedent(text): """Remove any common leading whitespace from every line in `text`. This can be used to make triple-quoted strings line up with the left edge of the display, while still presenting them in the source code in indented form. Note that tabs and spaces are both treated as whitespace, but they are not equal: the lines " hello" and "\\thello" are considered to have no common leading whitespace. (This behaviour is new in Python 2.5; older versions of this module incorrectly expanded tabs before searching for common leading whitespace.) """ # Look for the longest leading string of spaces and tabs common to # all lines. margin = None text = _whitespace_only_re.sub('', text) indents = _leading_whitespace_re.findall(text) for indent in indents: if margin is None: margin = indent # Current line more deeply indented than previous winner: # no change (previous winner is still on top). elif indent.startswith(margin): pass # Current line consistent with and no deeper than previous winner: # it's the new winner. elif margin.startswith(indent): margin = indent # Find the largest common whitespace between current line and previous # winner. else: for i, (x, y) in enumerate(zip(margin, indent)): if x != y: margin = margin[:i] break else: margin = margin[:len(indent)] # sanity check (testing/debugging only) if 0 and margin: for line in text.split("\n"): assert not line or line.startswith(margin), \ "line = %r, margin = %r" % (line, margin) if margin: text = re.sub(r'(?m)^' + margin, '', text) return text if __name__ == "__main__": #print dedent("\tfoo\n\tbar") #print dedent(" \thello there\n \t how are you?") print dedent("Hello there.\n This is indented.") ================================================ FILE: third_party/stdlib/threading.py ================================================ """Thread module emulating a subset of Java's threading model.""" import sys as _sys try: import thread except ImportError: del _sys.modules[__name__] raise import warnings from collections import deque as _deque from itertools import count as _count from time import time as _time, sleep as _sleep from traceback import format_exc as _format_exc # Note regarding PEP 8 compliant aliases # This threading model was originally inspired by Java, and inherited # the convention of camelCase function and method names from that # language. While those names are not in any imminent danger of being # deprecated, starting with Python 2.6, the module now provides a # PEP 8 compliant alias for any such method name. # Using the new PEP 8 compliant names also facilitates substitution # with the multiprocessing module, which doesn't provide the old # Java inspired names. # Rename some stuff so "from threading import *" is safe __all__ = ['activeCount', 'active_count', 'Condition', 'currentThread', 'current_thread', 'enumerate', 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Timer', 'setprofile', 'settrace', 'local', 'stack_size'] _start_new_thread = thread.start_new_thread _allocate_lock = thread.allocate_lock _get_ident = thread.get_ident ThreadError = thread.error del thread # sys.exc_clear is used to work around the fact that except blocks # don't fully clear the exception until 3.0. warnings.filterwarnings('ignore', category=DeprecationWarning, module='threading', message='sys.exc_clear') # Debug support (adapted from ihooks.py). # All the major classes here derive from _Verbose. We force that to # be a new-style class so that all the major classes here are new-style. # This helps debugging (type(instance) is more revealing for instances # of new-style classes). _VERBOSE = False if __debug__: class _Verbose(object): def __init__(self, verbose=None): if verbose is None: verbose = _VERBOSE self.__verbose = verbose def _note(self, format, *args): if self.__verbose: format = format % args # Issue #4188: calling current_thread() can incur an infinite # recursion if it has to create a DummyThread on the fly. ident = _get_ident() try: name = _active[ident].name except KeyError: name = "" % ident format = "%s: %s\n" % (name, format) _sys.stderr.write(format) else: # Disable this when using "python -O" class _Verbose(object): def __init__(self, verbose=None): pass def _note(self, *args): pass # Support for profile and trace hooks _profile_hook = None _trace_hook = None def setprofile(func): """Set a profile function for all threads started from the threading module. The func will be passed to sys.setprofile() for each thread, before its run() method is called. """ global _profile_hook _profile_hook = func def settrace(func): """Set a trace function for all threads started from the threading module. The func will be passed to sys.settrace() for each thread, before its run() method is called. """ global _trace_hook _trace_hook = func # Synchronization classes Lock = _allocate_lock def RLock(*args, **kwargs): """Factory function that returns a new reentrant lock. A reentrant lock must be released by the thread that acquired it. Once a thread has acquired a reentrant lock, the same thread may acquire it again without blocking; the thread must release it once for each time it has acquired it. """ return _RLock(*args, **kwargs) class _RLock(_Verbose): """A reentrant lock must be released by the thread that acquired it. Once a thread has acquired a reentrant lock, the same thread may acquire it again without blocking; the thread must release it once for each time it has acquired it. """ def __init__(self, verbose=None): _Verbose.__init__(self, verbose) self.__block = _allocate_lock() self.__owner = None self.__count = 0 def __repr__(self): owner = self.__owner try: owner = _active[owner].name except KeyError: pass return "<%s owner=%r count=%d>" % ( self.__class__.__name__, owner, self.__count) def acquire(self, blocking=1): """Acquire a lock, blocking or non-blocking. When invoked without arguments: if this thread already owns the lock, increment the recursion level by one, and return immediately. Otherwise, if another thread owns the lock, block until the lock is unlocked. Once the lock is unlocked (not owned by any thread), then grab ownership, set the recursion level to one, and return. If more than one thread is blocked waiting until the lock is unlocked, only one at a time will be able to grab ownership of the lock. There is no return value in this case. When invoked with the blocking argument set to true, do the same thing as when called without arguments, and return true. When invoked with the blocking argument set to false, do not block. If a call without an argument would block, return false immediately; otherwise, do the same thing as when called without arguments, and return true. """ me = _get_ident() if self.__owner == me: self.__count = self.__count + 1 if __debug__: self._note("%s.acquire(%s): recursive success", self, blocking) return 1 rc = self.__block.acquire(blocking) if rc: self.__owner = me self.__count = 1 if __debug__: self._note("%s.acquire(%s): initial success", self, blocking) else: if __debug__: self._note("%s.acquire(%s): failure", self, blocking) return rc __enter__ = acquire def release(self): """Release a lock, decrementing the recursion level. If after the decrement it is zero, reset the lock to unlocked (not owned by any thread), and if any other threads are blocked waiting for the lock to become unlocked, allow exactly one of them to proceed. If after the decrement the recursion level is still nonzero, the lock remains locked and owned by the calling thread. Only call this method when the calling thread owns the lock. A RuntimeError is raised if this method is called when the lock is unlocked. There is no return value. """ if self.__owner != _get_ident(): raise RuntimeError("cannot release un-acquired lock") self.__count = count = self.__count - 1 if not count: self.__owner = None self.__block.release() if __debug__: self._note("%s.release(): final release", self) else: if __debug__: self._note("%s.release(): non-final release", self) def __exit__(self, t, v, tb): self.release() # Internal methods used by condition variables def _acquire_restore(self, count_owner): count, owner = count_owner self.__block.acquire() self.__count = count self.__owner = owner if __debug__: self._note("%s._acquire_restore()", self) def _release_save(self): if __debug__: self._note("%s._release_save()", self) count = self.__count self.__count = 0 owner = self.__owner self.__owner = None self.__block.release() return (count, owner) def _is_owned(self): return self.__owner == _get_ident() def Condition(*args, **kwargs): """Factory function that returns a new condition variable object. A condition variable allows one or more threads to wait until they are notified by another thread. If the lock argument is given and not None, it must be a Lock or RLock object, and it is used as the underlying lock. Otherwise, a new RLock object is created and used as the underlying lock. """ return _Condition(*args, **kwargs) class _Condition(_Verbose): """Condition variables allow one or more threads to wait until they are notified by another thread. """ def __init__(self, lock=None, verbose=None): _Verbose.__init__(self, verbose) if lock is None: lock = RLock() self.__lock = lock # Export the lock's acquire() and release() methods self.acquire = lock.acquire self.release = lock.release # If the lock defines _release_save() and/or _acquire_restore(), # these override the default implementations (which just call # release() and acquire() on the lock). Ditto for _is_owned(). try: self._release_save = lock._release_save except AttributeError: pass try: self._acquire_restore = lock._acquire_restore except AttributeError: pass try: self._is_owned = lock._is_owned except AttributeError: pass self.__waiters = [] def __enter__(self): return self.__lock.__enter__() def __exit__(self, *args): return self.__lock.__exit__(*args) def __repr__(self): return "" % (self.__lock, len(self.__waiters)) def _release_save(self): self.__lock.release() # No state to save def _acquire_restore(self, x): self.__lock.acquire() # Ignore saved state def _is_owned(self): # Return True if lock is owned by current_thread. # This method is called only if __lock doesn't have _is_owned(). if self.__lock.acquire(0): self.__lock.release() return False else: return True def wait(self, timeout=None): """Wait until notified or until a timeout occurs. If the calling thread has not acquired the lock when this method is called, a RuntimeError is raised. This method releases the underlying lock, and then blocks until it is awakened by a notify() or notifyAll() call for the same condition variable in another thread, or until the optional timeout occurs. Once awakened or timed out, it re-acquires the lock and returns. When the timeout argument is present and not None, it should be a floating point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an RLock, it is not released using its release() method, since this may not actually unlock the lock when it was acquired multiple times recursively. Instead, an internal interface of the RLock class is used, which really unlocks it even when it has been recursively acquired several times. Another internal interface is then used to restore the recursion level when the lock is reacquired. """ if not self._is_owned(): raise RuntimeError("cannot wait on un-acquired lock") waiter = _allocate_lock() waiter.acquire() self.__waiters.append(waiter) saved_state = self._release_save() try: # restore state no matter what (e.g., KeyboardInterrupt) if timeout is None: waiter.acquire() if __debug__: self._note("%s.wait(): got it", self) else: # Balancing act: We can't afford a pure busy loop, so we # have to sleep; but if we sleep the whole timeout time, # we'll be unresponsive. The scheme here sleeps very # little at first, longer as time goes on, but never longer # than 20 times per second (or the timeout time remaining). endtime = _time() + timeout delay = 0.0005 # 500 us -> initial delay of 1 ms while True: gotit = waiter.acquire(0) if gotit: break remaining = endtime - _time() if remaining <= 0: break delay = min(delay * 2, remaining, .05) _sleep(delay) if not gotit: if __debug__: self._note("%s.wait(%s): timed out", self, timeout) try: self.__waiters.remove(waiter) except ValueError: pass else: if __debug__: self._note("%s.wait(%s): got it", self, timeout) finally: self._acquire_restore(saved_state) def notify(self, n=1): """Wake up one or more threads waiting on this condition, if any. If the calling thread has not acquired the lock when this method is called, a RuntimeError is raised. This method wakes up at most n of the threads waiting for the condition variable; it is a no-op if no threads are waiting. """ if not self._is_owned(): raise RuntimeError("cannot notify on un-acquired lock") __waiters = self.__waiters waiters = __waiters[:n] if not waiters: if __debug__: self._note("%s.notify(): no waiters", self) return self._note("%s.notify(): notifying %d waiter%s", self, n, n!=1 and "s" or "") for waiter in waiters: waiter.release() try: __waiters.remove(waiter) except ValueError: pass def notifyAll(self): """Wake up all threads waiting on this condition. If the calling thread has not acquired the lock when this method is called, a RuntimeError is raised. """ self.notify(len(self.__waiters)) notify_all = notifyAll def Semaphore(*args, **kwargs): """A factory function that returns a new semaphore. Semaphores manage a counter representing the number of release() calls minus the number of acquire() calls, plus an initial value. The acquire() method blocks if necessary until it can return without making the counter negative. If not given, value defaults to 1. """ return _Semaphore(*args, **kwargs) class _Semaphore(_Verbose): """Semaphores manage a counter representing the number of release() calls minus the number of acquire() calls, plus an initial value. The acquire() method blocks if necessary until it can return without making the counter negative. If not given, value defaults to 1. """ # After Tim Peters' semaphore class, but not quite the same (no maximum) def __init__(self, value=1, verbose=None): if value < 0: raise ValueError("semaphore initial value must be >= 0") _Verbose.__init__(self, verbose) self.__cond = Condition(Lock()) self.__value = value def acquire(self, blocking=1): """Acquire a semaphore, decrementing the internal counter by one. When invoked without arguments: if the internal counter is larger than zero on entry, decrement it by one and return immediately. If it is zero on entry, block, waiting until some other thread has called release() to make it larger than zero. This is done with proper interlocking so that if multiple acquire() calls are blocked, release() will wake exactly one of them up. The implementation may pick one at random, so the order in which blocked threads are awakened should not be relied on. There is no return value in this case. When invoked with blocking set to true, do the same thing as when called without arguments, and return true. When invoked with blocking set to false, do not block. If a call without an argument would block, return false immediately; otherwise, do the same thing as when called without arguments, and return true. """ rc = False with self.__cond: while self.__value == 0: if not blocking: break if __debug__: self._note("%s.acquire(%s): blocked waiting, value=%s", self, blocking, self.__value) self.__cond.wait() else: self.__value = self.__value - 1 if __debug__: self._note("%s.acquire: success, value=%s", self, self.__value) rc = True return rc __enter__ = acquire def release(self): """Release a semaphore, incrementing the internal counter by one. When the counter is zero on entry and another thread is waiting for it to become larger than zero again, wake up that thread. """ with self.__cond: self.__value = self.__value + 1 if __debug__: self._note("%s.release: success, value=%s", self, self.__value) self.__cond.notify() def __exit__(self, t, v, tb): self.release() def BoundedSemaphore(*args, **kwargs): """A factory function that returns a new bounded semaphore. A bounded semaphore checks to make sure its current value doesn't exceed its initial value. If it does, ValueError is raised. In most situations semaphores are used to guard resources with limited capacity. If the semaphore is released too many times it's a sign of a bug. If not given, value defaults to 1. Like regular semaphores, bounded semaphores manage a counter representing the number of release() calls minus the number of acquire() calls, plus an initial value. The acquire() method blocks if necessary until it can return without making the counter negative. If not given, value defaults to 1. """ return _BoundedSemaphore(*args, **kwargs) class _BoundedSemaphore(_Semaphore): """A bounded semaphore checks to make sure its current value doesn't exceed its initial value. If it does, ValueError is raised. In most situations semaphores are used to guard resources with limited capacity. """ def __init__(self, value=1, verbose=None): _Semaphore.__init__(self, value, verbose) self._initial_value = value def release(self): """Release a semaphore, incrementing the internal counter by one. When the counter is zero on entry and another thread is waiting for it to become larger than zero again, wake up that thread. If the number of releases exceeds the number of acquires, raise a ValueError. """ with self.__cond: if self.__value >= self._initial_value: raise ValueError("Semaphore released too many times") self.__value += 1 self.__cond.notify() def Event(*args, **kwargs): """A factory function that returns a new event. Events manage a flag that can be set to true with the set() method and reset to false with the clear() method. The wait() method blocks until the flag is true. """ return _Event(*args, **kwargs) class _Event(_Verbose): """A factory function that returns a new event object. An event manages a flag that can be set to true with the set() method and reset to false with the clear() method. The wait() method blocks until the flag is true. """ # After Tim Peters' event class (without is_posted()) def __init__(self, verbose=None): _Verbose.__init__(self, verbose) self.__cond = Condition(Lock()) self.__flag = False def _reset_internal_locks(self): # private! called by Thread._reset_internal_locks by _after_fork() self.__cond.__init__(Lock()) def isSet(self): 'Return true if and only if the internal flag is true.' return self.__flag is_set = isSet def set(self): """Set the internal flag to true. All threads waiting for the flag to become true are awakened. Threads that call wait() once the flag is true will not block at all. """ with self.__cond: self.__flag = True self.__cond.notify_all() def clear(self): """Reset the internal flag to false. Subsequently, threads calling wait() will block until set() is called to set the internal flag to true again. """ with self.__cond: self.__flag = False def wait(self, timeout=None): """Block until the internal flag is true. If the internal flag is true on entry, return immediately. Otherwise, block until another thread calls set() to set the flag to true, or until the optional timeout occurs. When the timeout argument is present and not None, it should be a floating point number specifying a timeout for the operation in seconds (or fractions thereof). This method returns the internal flag on exit, so it will always return True except if a timeout is given and the operation times out. """ with self.__cond: if not self.__flag: self.__cond.wait(timeout) return self.__flag # Helper to generate new thread names _counter = _count().next _counter() # Consume 0 so first non-main thread has id 1. def _newname(template="Thread-%d"): return template % _counter() # Active thread administration _active_limbo_lock = _allocate_lock() _active = {} # maps thread id to Thread object _limbo = {} # Main class for threads class Thread(_Verbose): """A class that represents a thread of control. This class can be safely subclassed in a limited fashion. """ __initialized = False def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, verbose=None): """This constructor should always be called with keyword arguments. Arguments are: *group* should be None; reserved for future extension when a ThreadGroup class is implemented. *target* is the callable object to be invoked by the run() method. Defaults to None, meaning nothing is called. *name* is the thread name. By default, a unique name is constructed of the form "Thread-N" where N is a small decimal number. *args* is the argument tuple for the target invocation. Defaults to (). *kwargs* is a dictionary of keyword arguments for the target invocation. Defaults to {}. If a subclass overrides the constructor, it must make sure to invoke the base class constructor (Thread.__init__()) before doing anything else to the thread. """ assert group is None, "group argument must be None for now" _Verbose.__init__(self, verbose) if kwargs is None: kwargs = {} self.__target = target self.__name = str(name or _newname()) self.__args = args self.__kwargs = kwargs self.__daemonic = self._set_daemon() self.__ident = None self.__started = Event() self.__stopped = False self.__block = Condition(Lock()) self.__initialized = True # sys.stderr is not stored in the class like # sys.exc_info since it can be changed between instances self.__stderr = _sys.stderr def _reset_internal_locks(self): # private! Called by _after_fork() to reset our internal locks as # they may be in an invalid state leading to a deadlock or crash. if hasattr(self, '__block'): # DummyThread deletes self.__block self.__block.__init__() self.__started._reset_internal_locks() @property def _block(self): # used by a unittest return self.__block def _set_daemon(self): # Overridden in _MainThread and _DummyThread return current_thread().daemon def __repr__(self): assert self.__initialized, "Thread.__init__() was not called" status = "initial" if self.__started.is_set(): status = "started" if self.__stopped: status = "stopped" if self.__daemonic: status += " daemon" if self.__ident is not None: status += " %s" % self.__ident return "<%s(%s, %s)>" % (self.__class__.__name__, self.__name, status) def start(self): """Start the thread's activity. It must be called at most once per thread object. It arranges for the object's run() method to be invoked in a separate thread of control. This method will raise a RuntimeError if called more than once on the same thread object. """ if not self.__initialized: raise RuntimeError("thread.__init__() not called") if self.__started.is_set(): raise RuntimeError("threads can only be started once") if __debug__: self._note("%s.start(): starting thread", self) with _active_limbo_lock: _limbo[self] = self try: _start_new_thread(self.__bootstrap, ()) except Exception: with _active_limbo_lock: del _limbo[self] raise self.__started.wait() def run(self): """Method representing the thread's activity. You may override this method in a subclass. The standard run() method invokes the callable object passed to the object's constructor as the target argument, if any, with sequential and keyword arguments taken from the args and kwargs arguments, respectively. """ try: if self.__target: self.__target(*self.__args, **self.__kwargs) finally: # Avoid a refcycle if the thread is running a function with # an argument that has a member that points to the thread. del self.__target, self.__args, self.__kwargs def __bootstrap(self): # Wrapper around the real bootstrap code that ignores # exceptions during interpreter cleanup. Those typically # happen when a daemon thread wakes up at an unfortunate # moment, finds the world around it destroyed, and raises some # random exception *** while trying to report the exception in # __bootstrap_inner() below ***. Those random exceptions # don't help anybody, and they confuse users, so we suppress # them. We suppress them only when it appears that the world # indeed has already been destroyed, so that exceptions in # __bootstrap_inner() during normal business hours are properly # reported. Also, we only suppress them for daemonic threads; # if a non-daemonic encounters this, something else is wrong. try: self.__bootstrap_inner() except: if self.__daemonic and _sys is None: return raise def _set_ident(self): self.__ident = _get_ident() def __bootstrap_inner(self): try: self._set_ident() self.__started.set() with _active_limbo_lock: _active[self.__ident] = self del _limbo[self] if __debug__: self._note("%s.__bootstrap(): thread started", self) if _trace_hook: self._note("%s.__bootstrap(): registering trace hook", self) _sys.settrace(_trace_hook) if _profile_hook: self._note("%s.__bootstrap(): registering profile hook", self) _sys.setprofile(_profile_hook) try: self.run() except SystemExit: if __debug__: self._note("%s.__bootstrap(): raised SystemExit", self) except: if __debug__: self._note("%s.__bootstrap(): unhandled exception", self) # If sys.stderr is no more (most likely from interpreter # shutdown) use self.__stderr. Otherwise still use sys (as in # _sys) in case sys.stderr was redefined since the creation of # self. if _sys and _sys.stderr is not None: print>>_sys.stderr, ("Exception in thread %s:\n%s" % (self.name, _format_exc())) elif self.__stderr is not None: # Do the best job possible w/o a huge amt. of code to # approximate a traceback (code ideas from # Lib/traceback.py) exc_type, exc_value, exc_tb = _sys.exc_info() try: print>>self.__stderr, ( "Exception in thread " + self.name + " (most likely raised during interpreter shutdown):") print>>self.__stderr, ( "Traceback (most recent call last):") while exc_tb: print>>self.__stderr, ( ' File "%s", line %s, in %s' % (exc_tb.tb_frame.f_code.co_filename, exc_tb.tb_lineno, exc_tb.tb_frame.f_code.co_name)) exc_tb = exc_tb.tb_next print>>self.__stderr, ("%s: %s" % (exc_type, exc_value)) # Make sure that exc_tb gets deleted since it is a memory # hog; deleting everything else is just for thoroughness finally: del exc_type, exc_value, exc_tb else: if __debug__: self._note("%s.__bootstrap(): normal return", self) finally: # Prevent a race in # test_threading.test_no_refcycle_through_target when # the exception keeps the target alive past when we # assert that it's dead. _sys.exc_clear() finally: with _active_limbo_lock: self.__stop() try: # We don't call self.__delete() because it also # grabs _active_limbo_lock. del _active[_get_ident()] except: pass def __stop(self): # DummyThreads delete self.__block, but they have no waiters to # notify anyway (join() is forbidden on them). if not hasattr(self, '__block'): return self.__block.acquire() self.__stopped = True self.__block.notify_all() self.__block.release() def __delete(self): "Remove current thread from the dict of currently running threads." # Notes about running with dummy_thread: # # Must take care to not raise an exception if dummy_thread is being # used (and thus this module is being used as an instance of # dummy_threading). dummy_thread.get_ident() always returns -1 since # there is only one thread if dummy_thread is being used. Thus # len(_active) is always <= 1 here, and any Thread instance created # overwrites the (if any) thread currently registered in _active. # # An instance of _MainThread is always created by 'threading'. This # gets overwritten the instant an instance of Thread is created; both # threads return -1 from dummy_thread.get_ident() and thus have the # same key in the dict. So when the _MainThread instance created by # 'threading' tries to clean itself up when atexit calls this method # it gets a KeyError if another Thread instance was created. # # This all means that KeyError from trying to delete something from # _active if dummy_threading is being used is a red herring. But # since it isn't if dummy_threading is *not* being used then don't # hide the exception. try: with _active_limbo_lock: del _active[_get_ident()] # There must not be any python code between the previous line # and after the lock is released. Otherwise a tracing function # could try to acquire the lock again in the same thread, (in # current_thread()), and would block. except KeyError: if 'dummy_threading' not in _sys.modules: raise def join(self, timeout=None): """Wait until the thread terminates. This blocks the calling thread until the thread whose join() method is called terminates -- either normally or through an unhandled exception or until the optional timeout occurs. When the timeout argument is present and not None, it should be a floating point number specifying a timeout for the operation in seconds (or fractions thereof). As join() always returns None, you must call isAlive() after join() to decide whether a timeout happened -- if the thread is still alive, the join() call timed out. When the timeout argument is not present or None, the operation will block until the thread terminates. A thread can be join()ed many times. join() raises a RuntimeError if an attempt is made to join the current thread as that would cause a deadlock. It is also an error to join() a thread before it has been started and attempts to do so raises the same exception. """ if not self.__initialized: raise RuntimeError("Thread.__init__() not called") if not self.__started.is_set(): raise RuntimeError("cannot join thread before it is started") if self is current_thread(): raise RuntimeError("cannot join current thread") if __debug__: if not self.__stopped: self._note("%s.join(): waiting until thread stops", self) self.__block.acquire() try: if timeout is None: while not self.__stopped: self.__block.wait() if __debug__: self._note("%s.join(): thread stopped", self) else: deadline = _time() + timeout while not self.__stopped: delay = deadline - _time() if delay <= 0: if __debug__: self._note("%s.join(): timed out", self) break self.__block.wait(delay) else: if __debug__: self._note("%s.join(): thread stopped", self) finally: self.__block.release() def _name_getter(self): """A string used for identification purposes only. It has no semantics. Multiple threads may be given the same name. The initial name is set by the constructor. """ assert self.__initialized, "Thread.__init__() not called" return self.__name def _name_setter(self, name): assert self.__initialized, "Thread.__init__() not called" self.__name = str(name) name = property(_name_getter, _name_setter) @property def ident(self): """Thread identifier of this thread or None if it has not been started. This is a nonzero integer. See the thread.get_ident() function. Thread identifiers may be recycled when a thread exits and another thread is created. The identifier is available even after the thread has exited. """ assert self.__initialized, "Thread.__init__() not called" return self.__ident def isAlive(self): """Return whether the thread is alive. This method returns True just before the run() method starts until just after the run() method terminates. The module function enumerate() returns a list of all alive threads. """ assert self.__initialized, "Thread.__init__() not called" return self.__started.is_set() and not self.__stopped is_alive = isAlive def _daemon_getter(self): """A boolean value indicating whether this thread is a daemon thread (True) or not (False). This must be set before start() is called, otherwise RuntimeError is raised. Its initial value is inherited from the creating thread; the main thread is not a daemon thread and therefore all threads created in the main thread default to daemon = False. The entire Python program exits when no alive non-daemon threads are left. """ assert self.__initialized, "Thread.__init__() not called" return self.__daemonic def _daemon_setter(self, daemonic): if not self.__initialized: raise RuntimeError("Thread.__init__() not called") if self.__started.is_set(): raise RuntimeError("cannot set daemon status of active thread"); self.__daemonic = daemonic daemon = property(_daemon_getter, _daemon_setter) def isDaemon(self): return self.daemon def setDaemon(self, daemonic): self.daemon = daemonic def getName(self): return self.name def setName(self, name): self.name = name # The timer class was contributed by Itamar Shtull-Trauring def Timer(*args, **kwargs): """Factory function to create a Timer object. Timers call a function after a specified number of seconds: t = Timer(30.0, f, args=[], kwargs={}) t.start() t.cancel() # stop the timer's action if it's still waiting """ return _Timer(*args, **kwargs) class _Timer(Thread): """Call a function after a specified number of seconds: t = Timer(30.0, f, args=[], kwargs={}) t.start() t.cancel() # stop the timer's action if it's still waiting """ def __init__(self, interval, function, args=[], kwargs={}): Thread.__init__(self) self.interval = interval self.function = function self.args = args self.kwargs = kwargs self.finished = Event() def cancel(self): """Stop the timer if it hasn't finished yet""" self.finished.set() def run(self): self.finished.wait(self.interval) if not self.finished.is_set(): self.function(*self.args, **self.kwargs) self.finished.set() # Special thread class to represent the main thread # This is garbage collected through an exit handler class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread") self.__started.set() self._set_ident() with _active_limbo_lock: _active[_get_ident()] = self def _set_daemon(self): return False def _exitfunc(self): self.__stop() t = _pickSomeNonDaemonThread() if t: if __debug__: self._note("%s: waiting for other threads", self) while t: t.join() t = _pickSomeNonDaemonThread() if __debug__: self._note("%s: exiting", self) self.__delete() def _pickSomeNonDaemonThread(): for t in enumerate(): if not t.daemon and t.is_alive(): return t return None # Dummy thread class to represent threads not started here. # These aren't garbage collected when they die, nor can they be waited for. # If they invoke anything in threading.py that calls current_thread(), they # leave an entry in the _active dict forever after. # Their purpose is to return *something* from current_thread(). # They are marked as daemon threads so we won't wait for them # when we exit (conform previous semantics). class _DummyThread(Thread): def __init__(self): Thread.__init__(self, name=_newname("Dummy-%d")) # Thread.__block consumes an OS-level locking primitive, which # can never be used by a _DummyThread. Since a _DummyThread # instance is immortal, that's bad, so release this resource. del self.__block self.__started.set() self._set_ident() with _active_limbo_lock: _active[_get_ident()] = self def _set_daemon(self): return True def join(self, timeout=None): assert False, "cannot join a dummy thread" # Global API functions def currentThread(): """Return the current Thread object, corresponding to the caller's thread of control. If the caller's thread of control was not created through the threading module, a dummy thread object with limited functionality is returned. """ try: return _active[_get_ident()] except KeyError: ##print "current_thread(): no current thread for", _get_ident() return _DummyThread() current_thread = currentThread def activeCount(): """Return the number of Thread objects currently alive. The returned count is equal to the length of the list returned by enumerate(). """ with _active_limbo_lock: return len(_active) + len(_limbo) active_count = activeCount def _enumerate(): # Same as enumerate(), but without the lock. Internal use only. return _active.values() + _limbo.values() def enumerate(): """Return a list of all Thread objects currently alive. The list includes daemonic threads, dummy thread objects created by current_thread(), and the main thread. It excludes terminated threads and threads that have not yet been started. """ with _active_limbo_lock: return _active.values() + _limbo.values() from thread import stack_size # Create the main thread object, # and make it available for the interpreter # (Py_Main) as threading._shutdown. _shutdown = _MainThread()._exitfunc # get thread-local implementation, either from the thread # module, or from the python fallback # NOTE: Thread local classes follow: the Grumpy version of this file copies # these from _threading_local.py to avoid circular dependency issues. class _localbase(object): __slots__ = '_local__key', '_local__args', '_local__lock' def __new__(cls, *args, **kw): self = object.__new__(cls) key = '_local__key', 'thread.local.' + str(id(self)) object.__setattr__(self, '_local__key', key) object.__setattr__(self, '_local__args', (args, kw)) object.__setattr__(self, '_local__lock', RLock()) if (args or kw) and (cls.__init__ is object.__init__): raise TypeError("Initialization arguments are not supported") # We need to create the thread dict in anticipation of # __init__ being called, to make sure we don't call it # again ourselves. dict = object.__getattribute__(self, '__dict__') current_thread().__dict__[key] = dict return self def _patch(self): key = object.__getattribute__(self, '_local__key') d = current_thread().__dict__.get(key) if d is None: d = {} current_thread().__dict__[key] = d object.__setattr__(self, '__dict__', d) # we have a new instance dict, so call out __init__ if we have # one cls = type(self) if cls.__init__ is not object.__init__: args, kw = object.__getattribute__(self, '_local__args') cls.__init__(self, *args, **kw) else: object.__setattr__(self, '__dict__', d) class local(_localbase): def __getattribute__(self, name): lock = object.__getattribute__(self, '_local__lock') lock.acquire() try: _patch(self) return object.__getattribute__(self, name) finally: lock.release() def __setattr__(self, name, value): if name == '__dict__': raise AttributeError( "%r object attribute '__dict__' is read-only" % self.__class__.__name__) lock = object.__getattribute__(self, '_local__lock') lock.acquire() try: _patch(self) return object.__setattr__(self, name, value) finally: lock.release() def __delattr__(self, name): if name == '__dict__': raise AttributeError( "%r object attribute '__dict__' is read-only" % self.__class__.__name__) lock = object.__getattribute__(self, '_local__lock') lock.acquire() try: _patch(self) return object.__delattr__(self, name) finally: lock.release() def __del__(self): key = object.__getattribute__(self, '_local__key') try: # We use the non-locking API since we might already hold the lock # (__del__ can be called at any point by the cyclic GC). threads = _enumerate() except: # If enumerating the current threads fails, as it seems to do # during shutdown, we'll skip cleanup under the assumption # that there is nothing to clean up. return for thread in threads: try: __dict__ = thread.__dict__ except AttributeError: # Thread is dying, rest in peace. continue if key in __dict__: try: del __dict__[key] except KeyError: pass # didn't have anything in this thread # END _threading_local.py copy def _after_fork(): # This function is called by Python/ceval.c:PyEval_ReInitThreads which # is called from PyOS_AfterFork. Here we cleanup threading module state # that should not exist after a fork. # Reset _active_limbo_lock, in case we forked while the lock was held # by another (non-forked) thread. http://bugs.python.org/issue874900 global _active_limbo_lock _active_limbo_lock = _allocate_lock() # fork() only copied the current thread; clear references to others. new_active = {} current = current_thread() with _active_limbo_lock: for thread in _enumerate(): # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. if hasattr(thread, '_reset_internal_locks'): thread._reset_internal_locks() if thread is current: # There is only one active thread. We reset the ident to # its new value since it can have changed. ident = _get_ident() thread.__ident = ident new_active[ident] = thread else: # All the others are already stopped. thread.__stop() _limbo.clear() _active.clear() _active.update(new_active) assert len(_active) == 1 # Self-test code def _test(): class BoundedQueue(_Verbose): def __init__(self, limit): _Verbose.__init__(self) self.mon = RLock() self.rc = Condition(self.mon) self.wc = Condition(self.mon) self.limit = limit self.queue = _deque() def put(self, item): self.mon.acquire() while len(self.queue) >= self.limit: self._note("put(%s): queue full", item) self.wc.wait() self.queue.append(item) self._note("put(%s): appended, length now %d", item, len(self.queue)) self.rc.notify() self.mon.release() def get(self): self.mon.acquire() while not self.queue: self._note("get(): queue empty") self.rc.wait() item = self.queue.popleft() self._note("get(): got %s, %d left", item, len(self.queue)) self.wc.notify() self.mon.release() return item class ProducerThread(Thread): def __init__(self, queue, quota): Thread.__init__(self, name="Producer") self.queue = queue self.quota = quota def run(self): from random import random counter = 0 while counter < self.quota: counter = counter + 1 self.queue.put("%s.%d" % (self.name, counter)) _sleep(random() * 0.00001) class ConsumerThread(Thread): def __init__(self, queue, count): Thread.__init__(self, name="Consumer") self.queue = queue self.count = count def run(self): while self.count > 0: item = self.queue.get() print item self.count = self.count - 1 NP = 3 QL = 4 NI = 5 Q = BoundedQueue(QL) P = [] for i in range(NP): t = ProducerThread(Q, NI) t.name = ("Producer-%d" % (i+1)) P.append(t) C = ConsumerThread(Q, NI*NP) for t in P: t.start() _sleep(0.000001) C.start() for t in P: t.join() C.join() if __name__ == '__main__': _test() ================================================ FILE: third_party/stdlib/traceback.py ================================================ """Extract, format and print information about Python stack traces.""" import linecache import sys import types __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', 'format_tb', 'print_exc', 'format_exc', 'print_exception', 'print_last', 'print_stack', 'print_tb', 'tb_lineno'] def _print(file, str='', terminator='\n'): file.write(str+terminator) def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or extract_stack() as a formatted stack trace to the given file.""" if file is None: file = sys.stderr for filename, lineno, name, line in extracted_list: _print(file, ' File "%s", line %d, in %s' % (filename,lineno,name)) if line: _print(file, ' %s' % line.strip()) def format_list(extracted_list): """Format a list of traceback entry tuples for printing. Given a list of tuples as returned by extract_tb() or extract_stack(), return a list of strings ready for printing. Each string in the resulting list corresponds to the item with the same index in the argument list. Each string ends in a newline; the strings may contain internal newlines as well, for those items whose source text line is not None. """ list = [] for filename, lineno, name, line in extracted_list: item = ' File "%s", line %d, in %s\n' % (filename,lineno,name) if line: item = item + ' %s\n' % line.strip() list.append(item) return list def print_tb(tb, limit=None, file=None): """Print up to 'limit' stack trace entries from the traceback 'tb'. If 'limit' is omitted or None, all entries are printed. If 'file' is omitted or None, the output goes to sys.stderr; otherwise 'file' should be an open file or file-like object with a write() method. """ if file is None: file = sys.stderr if limit is None: if hasattr(sys, 'tracebacklimit'): limit = sys.tracebacklimit n = 0 while tb is not None and (limit is None or n < limit): f = tb.tb_frame lineno = tb.tb_lineno co = f.f_code filename = co.co_filename name = co.co_name _print(file, ' File "%s", line %d, in %s' % (filename, lineno, name)) linecache.checkcache(filename) line = linecache.getline(filename, lineno, f.f_globals) if line: _print(file, ' ' + line.strip()) tb = tb.tb_next n = n+1 def format_tb(tb, limit = None): """A shorthand for 'format_list(extract_tb(tb, limit))'.""" return format_list(extract_tb(tb, limit)) def extract_tb(tb, limit = None): """Return list of up to limit pre-processed entries from traceback. This is useful for alternate formatting of stack traces. If 'limit' is omitted or None, all entries are extracted. A pre-processed stack trace entry is a quadruple (filename, line number, function name, text) representing the information that is usually printed for a stack trace. The text is a string with leading and trailing whitespace stripped; if the source is not available it is None. """ if limit is None: if hasattr(sys, 'tracebacklimit'): limit = sys.tracebacklimit list = [] n = 0 while tb is not None and (limit is None or n < limit): f = tb.tb_frame lineno = tb.tb_lineno co = f.f_code filename = co.co_filename name = co.co_name linecache.checkcache(filename) line = linecache.getline(filename, lineno, f.f_globals) if line: line = line.strip() else: line = None list.append((filename, lineno, name, line)) tb = tb.tb_next n = n+1 return list def print_exception(etype, value, tb, limit=None, file=None): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if traceback is not None, it prints a header "Traceback (most recent call last):"; (2) it prints the exception type and value after the stack trace; (3) if type is SyntaxError and value has the appropriate format, it prints the line where the syntax error occurred with a caret on the next line indicating the approximate position of the error. """ if file is None: # TODO: Use sys.stderr when that's implemented. file = open('/dev/stderr', 'w') #file = sys.stderr if tb: _print(file, 'Traceback (most recent call last):') print_tb(tb, limit, file) lines = format_exception_only(etype, value) for line in lines: _print(file, line, '') def format_exception(etype, value, tb, limit = None): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments to print_exception(). The return value is a list of strings, each ending in a newline and some containing internal newlines. When these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ if tb: list = ['Traceback (most recent call last):\n'] list = list + format_tb(tb, limit) else: list = [] list = list + format_exception_only(etype, value) return list def format_exception_only(etype, value): """Format the exception part of a traceback. The arguments are the exception type and value such as given by sys.last_type and sys.last_value. The return value is a list of strings, each ending in a newline. Normally, the list contains a single string; however, for SyntaxError exceptions, it contains several lines that (when printed) display detailed information about where the syntax error occurred. The message indicating which exception occurred is always the last string in the list. """ # An instance should not have a meaningful value parameter, but # sometimes does, particularly for string exceptions, such as # >>> raise string1, string2 # deprecated # # Clear these out first because issubtype(string1, SyntaxError) # would raise another exception and mask the original problem. if (isinstance(etype, BaseException) or # isinstance(etype, types.InstanceType) or etype is None or type(etype) is str): return [_format_final_exc_line(etype, value)] stype = etype.__name__ if not issubclass(etype, SyntaxError): return [_format_final_exc_line(stype, value)] # It was a syntax error; show exactly where the problem was found. lines = [] try: msg, (filename, lineno, offset, badline) = value.args except Exception: pass else: filename = filename or "" lines.append(' File "%s", line %d\n' % (filename, lineno)) if badline is not None: lines.append(' %s\n' % badline.strip()) if offset is not None: caretspace = badline.rstrip('\n') offset = min(len(caretspace), offset) - 1 caretspace = caretspace[:offset].lstrip() # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c.isspace() and c or ' ') for c in caretspace) lines.append(' %s^\n' % ''.join(caretspace)) value = msg lines.append(_format_final_exc_line(stype, value)) return lines def _format_final_exc_line(etype, value): """Return a list of a single line -- normal case for format_exception_only""" valuestr = _some_str(value) if value is None or not valuestr: line = "%s\n" % etype else: line = "%s: %s\n" % (etype, valuestr) return line def _some_str(value): try: return str(value) except Exception: pass try: value = unicode(value) return value.encode("ascii", "backslashreplace") except Exception: pass return '' % type(value).__name__ def print_exc(limit=None, file=None): """Shorthand for 'print_exception(sys.exc_type, sys.exc_value, sys.exc_traceback, limit, file)'. (In fact, it uses sys.exc_info() to retrieve the same information in a thread-safe way.)""" if file is None: # TODO: Use sys.stderr when that's implemented. file = open('/dev/stderr', 'w') #file = sys.stderr try: etype, value, tb = sys.exc_info() print_exception(etype, value, tb, limit, file) finally: etype = value = tb = None def format_exc(limit=None): """Like print_exc() but return a string.""" try: etype, value, tb = sys.exc_info() return ''.join(format_exception(etype, value, tb, limit)) finally: etype = value = tb = None def print_last(limit=None, file=None): """This is a shorthand for 'print_exception(sys.last_type, sys.last_value, sys.last_traceback, limit, file)'.""" if not hasattr(sys, "last_type"): raise ValueError("no last exception") if file is None: file = sys.stderr print_exception(sys.last_type, sys.last_value, sys.last_traceback, limit, file) def print_stack(f=None, limit=None, file=None): """Print a stack trace from its invocation point. The optional 'f' argument can be used to specify an alternate stack frame at which to start. The optional 'limit' and 'file' arguments have the same meaning as for print_exception(). """ if f is None: try: raise ZeroDivisionError except ZeroDivisionError: f = sys.exc_info()[2].tb_frame.f_back print_list(extract_stack(f, limit), file) def format_stack(f=None, limit=None): """Shorthand for 'format_list(extract_stack(f, limit))'.""" if f is None: try: raise ZeroDivisionError except ZeroDivisionError: f = sys.exc_info()[2].tb_frame.f_back return format_list(extract_stack(f, limit)) def extract_stack(f=None, limit = None): """Extract the raw traceback from the current stack frame. The return value has the same format as for extract_tb(). The optional 'f' and 'limit' arguments have the same meaning as for print_stack(). Each item in the list is a quadruple (filename, line number, function name, text), and the entries are in order from oldest to newest stack frame. """ if f is None: try: raise ZeroDivisionError except ZeroDivisionError: f = sys.exc_info()[2].tb_frame.f_back if limit is None: if hasattr(sys, 'tracebacklimit'): limit = sys.tracebacklimit list = [] n = 0 while f is not None and (limit is None or n < limit): lineno = f.f_lineno co = f.f_code filename = co.co_filename name = co.co_name linecache.checkcache(filename) line = linecache.getline(filename, lineno, f.f_globals) if line: line = line.strip() else: line = None list.append((filename, lineno, name, line)) f = f.f_back n = n+1 list.reverse() return list def tb_lineno(tb): """Calculate correct line number of traceback given in tb. Obsolete in 2.3. """ return tb.tb_lineno ================================================ FILE: third_party/stdlib/types.py ================================================ """Define names for all type symbols known in the standard interpreter. Types that are part of optional modules (e.g. array) are not listed. """ import sys # Iterators in Python aren't a matter of type but of protocol. A large # and changing number of builtin types implement *some* flavor of # iterator. Don't check the type! Use hasattr to check for both # "__iter__" and "next" attributes instead. NoneType = type(None) TypeType = type ObjectType = object IntType = int #LongType = long FloatType = float BooleanType = bool try: ComplexType = complex except NameError: pass StringType = str # StringTypes is already outdated. Instead of writing "type(x) in # types.StringTypes", you should use "isinstance(x, basestring)". But # we keep around for compatibility with Python 2.2. try: UnicodeType = unicode StringTypes = (StringType, UnicodeType) except NameError: StringTypes = (StringType,) #BufferType = buffer TupleType = tuple ListType = list DictType = DictionaryType = dict def _f(): pass FunctionType = type(_f) #LambdaType = type(lambda: None) # Same as FunctionType #CodeType = type(_f.func_code) def _g(): yield 1 GeneratorType = type(_g()) class _C(object): def _m(self): pass ClassType = type(_C) UnboundMethodType = type(_C._m) # Same as MethodType _x = _C() #InstanceType = type(_x) MethodType = type(_x._m) BuiltinFunctionType = type(len) BuiltinMethodType = type([].append) # Same as BuiltinFunctionType ModuleType = type(sys) FileType = file XRangeType = xrange try: raise TypeError except TypeError: tb = sys.exc_info()[2] TracebackType = type(tb) FrameType = type(tb.tb_frame) del tb SliceType = slice EllipsisType = type(Ellipsis) #DictProxyType = type(TypeType.__dict__) NotImplementedType = type(NotImplemented) # For Jython, the following two types are identical #GetSetDescriptorType = type(FunctionType.func_code) #MemberDescriptorType = type(FunctionType.func_globals) del sys, _C, _x # Not for export #del _f, _g ================================================ FILE: third_party/stdlib/unittest/__init__.py ================================================ """ Python unit testing framework, based on Erich Gamma's JUnit and Kent Beck's Smalltalk testing framework. This module contains the core framework classes that form the basis of specific test cases and suites (TestCase, TestSuite etc.), and also a text-based utility class for running the tests and reporting the results (TextTestRunner). Simple usage: import unittest class IntegerArithmeticTestCase(unittest.TestCase): def testAdd(self): ## test method names begin 'test*' self.assertEqual((1 + 2), 3) self.assertEqual(0 + 1, 1) def testMultiply(self): self.assertEqual((0 * 10), 0) self.assertEqual((5 * 8), 40) if __name__ == '__main__': unittest.main() Further information is available in the bundled documentation, and from http://docs.python.org/library/unittest.html Copyright (c) 1999-2003 Steve Purcell Copyright (c) 2003-2010 Python Software Foundation This module is free software, and you may redistribute it and/or modify it under the same terms as Python itself, so long as this copyright message and disclaimer are retained in their original form. IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. """ __all__ = ['TestResult', 'TestCase', 'TestSuite', 'TextTestRunner', 'TestLoader', 'FunctionTestCase', 'defaultTestLoader', 'SkipTest', 'skip', 'skipIf', 'skipUnless', 'expectedFailure', 'TextTestResult', 'installHandler', 'registerResult', 'removeResult', 'removeHandler'] # Expose obsolete functions for backwards compatibility # __all__.extend(['getTestCaseNames', 'makeSuite', 'findTestCases']) __all__ += (['getTestCaseNames', 'makeSuite', 'findTestCases']) __unittest = True import unittest_result import unittest_case import unittest_suite import unittest_loader # import unittest_main import unittest_runner import unittest_signals # from .result import TestResult # from .case import (TestCase, FunctionTestCase, SkipTest, skip, skipIf, # skipUnless, expectedFailure) # from .suite import BaseTestSuite, TestSuite # from .loader import (TestLoader, defaultTestLoader, makeSuite, getTestCaseNames, # findTestCases) # from .main import TestProgram, main # from .runner import TextTestRunner, TextTestResult # from .signals import installHandler, registerResult, removeResult, removeHandler TestResult = unittest_result.TestResult TestCase, FunctionTestCase, SkipTest, skip, skipIf, skipUnless, expectedFailure = \ unittest_case.TestCase, unittest_case.FunctionTestCase, unittest_case.SkipTest, \ unittest_case.skip, unittest_case.skipIf, unittest_case.skipUnless, \ unittest_case.expectedFailure BaseTestSuite, TestSuite = unittest_suite.BaseTestSuite, unittest_suite.TestSuite TestLoader, defaultTestLoader, makeSuite, getTestCaseNames, findTestCases = \ unittest_loader.TestLoader, unittest_loader.defaultTestLoader, unittest_loader.makeSuite, \ unittest_loader.getTestCaseNames, unittest_loader.findTestCases # TestProgram, main = unittest_main.TestProgram, unittest_main.main TextTestRunner, TextTestResult = unittest_runner.TextTestRunner, unittest_runner.TextTestResult installHandler, registerResult, removeResult, removeHandler = \ unittest_signals.installHandler, unittest_signals.registerResult, \ unittest_signals.removeResult, unittest_signals.removeHandler # deprecated _TextTestResult = TextTestResult ================================================ FILE: third_party/stdlib/unittest_case.py ================================================ """Test case implementation""" import collections import sys import functools import difflib import pprint import re import types import warnings # from . import result import unittest_result as result import unittest_util as _util # from .util import ( # strclass, safe_repr, unorderable_list_difference, # _count_diff_all_purpose, _count_diff_hashable # ) strclass, safe_repr, unorderable_list_difference, _count_diff_all_purpose, \ _count_diff_hashable = _util.strclass, _util.safe_repr, \ _util.unorderable_list_difference, _util._count_diff_all_purpose, \ _util._count_diff_hashable class KeyboardInterrupt(BaseException): pass __unittest = True DIFF_OMITTED = ('\nDiff is %s characters long. ' 'Set self.maxDiff to None to see it.') class SkipTest(Exception): """ Raise this exception in a test to skip it. Usually you can use TestCase.skipTest() or one of the skipping decorators instead of raising this directly. """ pass class _ExpectedFailure(Exception): """ Raise this when a test is expected to fail. This is an implementation detail. """ def __init__(self, exc_info): super(_ExpectedFailure, self).__init__() self.exc_info = exc_info class _UnexpectedSuccess(Exception): """ The test was supposed to fail, but it didn't! """ pass def _id(obj): return obj def skip(reason): """ Unconditionally skip a test. """ def decorator(test_item): if not isinstance(test_item, (type, types.ClassType)): # @functools.wraps(test_item) def skip_wrapper(*args, **kwargs): raise SkipTest(reason) skip_wrapper = functools.wraps(test_item)(skip_wrapper) test_item = skip_wrapper test_item.__unittest_skip__ = True test_item.__unittest_skip_why__ = reason return test_item return decorator def skipIf(condition, reason): """ Skip a test if the condition is true. """ if condition: return skip(reason) return _id def skipUnless(condition, reason): """ Skip a test unless the condition is true. """ if not condition: return skip(reason) return _id def expectedFailure(func): # @functools.wraps(func) def wrapper(*args, **kwargs): try: func(*args, **kwargs) except Exception: raise _ExpectedFailure(sys.exc_info()) raise _UnexpectedSuccess wrapper = functools.wraps(func)(wrapper) return wrapper class _AssertRaisesContext(object): """A context manager used to implement TestCase.assertRaises* methods.""" def __init__(self, expected, test_case, expected_regexp=None): self.expected = expected self.failureException = test_case.failureException self.expected_regexp = expected_regexp def __enter__(self): return self def __exit__(self, exc_type, exc_value, tb): if exc_type is None: try: exc_name = self.expected.__name__ except AttributeError: exc_name = str(self.expected) raise self.failureException( # "{0} not raised".format(exc_name)) "%s not raised" % (exc_name)) if not issubclass(exc_type, self.expected): # let unexpected exceptions pass through return False self.exception = exc_value # store for later retrieval if self.expected_regexp is None: return True expected_regexp = self.expected_regexp if not expected_regexp.search(str(exc_value)): raise self.failureException('"%s" does not match "%s"' % (expected_regexp.pattern, str(exc_value))) return True class TestCase(object): """A class whose instances are single test cases. By default, the test code itself should be placed in a method named 'runTest'. If the fixture may be used for many test cases, create as many test methods as are needed. When instantiating such a TestCase subclass, specify in the constructor arguments the name of the test method that the instance is to execute. Test authors should subclass TestCase for their own tests. Construction and deconstruction of the test's environment ('fixture') can be implemented by overriding the 'setUp' and 'tearDown' methods respectively. If it is necessary to override the __init__ method, the base class __init__ method must always be called. It is important that subclasses should not change the signature of their __init__ method, since instances of the classes are instantiated automatically by parts of the framework in order to be run. When subclassing TestCase, you can set these attributes: * failureException: determines which exception will be raised when the instance's assertion methods fail; test methods raising this exception will be deemed to have 'failed' rather than 'errored'. * longMessage: determines whether long messages (including repr of objects used in assert methods) will be printed on failure in *addition* to any explicit message passed. * maxDiff: sets the maximum length of a diff in failure messages by assert methods using difflib. It is looked up as an instance attribute so can be configured by individual tests if required. """ failureException = AssertionError longMessage = False maxDiff = 80*8 # If a string is longer than _diffThreshold, use normal comparison instead # of difflib. See #11763. # _diffThreshold = 2**16 _diffThreshold = 1<<16 # Attribute used by TestSuite for classSetUp _classSetupFailed = False def __init__(self, methodName='runTest'): """Create an instance of the class that will use the named test method when executed. Raises a ValueError if the instance does not have a method with the specified name. """ self._testMethodName = methodName self._resultForDoCleanups = None try: testMethod = getattr(self, methodName) except AttributeError: raise ValueError("no such test method in %s: %s" % (self.__class__, methodName)) # self._testMethodDoc = testMethod.__doc__ self._cleanups = [] # Map types to custom assertEqual functions that will compare # instances of said type in more detail to generate a more useful # error message. self._type_equality_funcs = {} self.addTypeEqualityFunc(dict, 'assertDictEqual') self.addTypeEqualityFunc(list, 'assertListEqual') self.addTypeEqualityFunc(tuple, 'assertTupleEqual') self.addTypeEqualityFunc(set, 'assertSetEqual') self.addTypeEqualityFunc(frozenset, 'assertSetEqual') try: self.addTypeEqualityFunc(unicode, 'assertMultiLineEqual') except NameError: # No unicode support in this build pass def addTypeEqualityFunc(self, typeobj, function): """Add a type specific assertEqual style function to compare a type. This method is for use by TestCase subclasses that need to register their own type equality functions to provide nicer error messages. Args: typeobj: The data type to call this function on when both values are of the same type in assertEqual(). function: The callable taking two arguments and an optional msg= argument that raises self.failureException with a useful error message when the two arguments are not equal. """ self._type_equality_funcs[typeobj] = function def addCleanup(self, function, *args, **kwargs): """Add a function, with arguments, to be called when the test is completed. Functions added are called on a LIFO basis and are called after tearDown on test failure or success. Cleanup items are called even if setUp fails (unlike tearDown).""" self._cleanups.append((function, args, kwargs)) def setUp(self): "Hook method for setting up the test fixture before exercising it." pass def tearDown(self): "Hook method for deconstructing the test fixture after testing it." pass # @classmethod def setUpClass(cls): "Hook method for setting up class fixture before running tests in the class." setUpClass = classmethod(setUpClass) # @classmethod def tearDownClass(cls): "Hook method for deconstructing the class fixture after running all tests in the class." tearDownClass = classmethod(tearDownClass) def countTestCases(self): return 1 def defaultTestResult(self): return result.TestResult() def shortDescription(self): """Returns a one-line description of the test, or None if no description has been provided. The default implementation of this method returns the first line of the specified test method's docstring. """ # doc = self._testMethodDoc # return doc and doc.split("\n")[0].strip() or None return '' def id(self): return "%s.%s" % (strclass(self.__class__), self._testMethodName) def __eq__(self, other): if type(self) is not type(other): return NotImplemented return self._testMethodName == other._testMethodName def __ne__(self, other): return not self == other def __hash__(self): return hash((type(self), self._testMethodName)) def __str__(self): return "%s (%s)" % (self._testMethodName, strclass(self.__class__)) def __repr__(self): return "<%s testMethod=%s>" % \ (strclass(self.__class__), self._testMethodName) def _addSkip(self, result, reason): addSkip = getattr(result, 'addSkip', None) if addSkip is not None: addSkip(self, reason) else: warnings.warn("TestResult has no addSkip method, skips not reported", RuntimeWarning, 2) result.addSuccess(self) def run(self, result=None): orig_result = result if result is None: result = self.defaultTestResult() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() self._resultForDoCleanups = result result.startTest(self) testMethod = getattr(self, self._testMethodName) if (getattr(self.__class__, "__unittest_skip__", False) or getattr(testMethod, "__unittest_skip__", False)): # If the class or method was skipped. try: skip_why = (getattr(self.__class__, '__unittest_skip_why__', '') or getattr(testMethod, '__unittest_skip_why__', '')) self._addSkip(result, skip_why) finally: result.stopTest(self) return try: success = False try: self.setUp() except SkipTest as e: self._addSkip(result, str(e)) except KeyboardInterrupt: raise except: result.addError(self, sys.exc_info()) else: try: testMethod() except KeyboardInterrupt: raise except self.failureException: result.addFailure(self, sys.exc_info()) except _ExpectedFailure as e: addExpectedFailure = getattr(result, 'addExpectedFailure', None) if addExpectedFailure is not None: addExpectedFailure(self, e.exc_info) else: warnings.warn("TestResult has no addExpectedFailure method, reporting as passes", RuntimeWarning) result.addSuccess(self) except _UnexpectedSuccess: addUnexpectedSuccess = getattr(result, 'addUnexpectedSuccess', None) if addUnexpectedSuccess is not None: addUnexpectedSuccess(self) else: warnings.warn("TestResult has no addUnexpectedSuccess method, reporting as failures", RuntimeWarning) result.addFailure(self, sys.exc_info()) except SkipTest as e: self._addSkip(result, str(e)) except: result.addError(self, sys.exc_info()) else: success = True try: self.tearDown() except KeyboardInterrupt: raise except: result.addError(self, sys.exc_info()) success = False cleanUpSuccess = self.doCleanups() success = success and cleanUpSuccess if success: result.addSuccess(self) finally: result.stopTest(self) if orig_result is None: stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() def doCleanups(self): """Execute all cleanup functions. Normally called for you after tearDown.""" result = self._resultForDoCleanups ok = True while self._cleanups: function, args, kwargs = self._cleanups.pop(-1) try: function(*args, **kwargs) except KeyboardInterrupt: raise except: ok = False result.addError(self, sys.exc_info()) return ok def __call__(self, *args, **kwds): return self.run(*args, **kwds) def debug(self): """Run the test without collecting errors in a TestResult""" self.setUp() getattr(self, self._testMethodName)() self.tearDown() while self._cleanups: function, args, kwargs = self._cleanups.pop(-1) function(*args, **kwargs) def skipTest(self, reason): """Skip this test.""" raise SkipTest(reason) def fail(self, msg=None): """Fail immediately, with the given message.""" raise self.failureException(msg) def assertFalse(self, expr, msg=None): """Check that the expression is false.""" if expr: msg = self._formatMessage(msg, "%s is not false" % safe_repr(expr)) raise self.failureException(msg) def assertTrue(self, expr, msg=None): """Check that the expression is true.""" if not expr: msg = self._formatMessage(msg, "%s is not true" % safe_repr(expr)) raise self.failureException(msg) def _formatMessage(self, msg, standardMsg): """Honour the longMessage attribute when generating failure messages. If longMessage is False this means: * Use only an explicit message if it is provided * Otherwise use the standard message for the assert If longMessage is True: * Use the standard message * If an explicit message is provided, plus ' : ' and the explicit message """ if not self.longMessage: return msg or standardMsg if msg is None: return standardMsg try: # don't switch to '{}' formatting in Python 2.X # it changes the way unicode input is handled return '%s : %s' % (standardMsg, msg) except UnicodeDecodeError: return '%s : %s' % (safe_repr(standardMsg), safe_repr(msg)) def assertRaises(self, excClass, callableObj=None, *args, **kwargs): """Fail unless an exception of class excClass is raised by callableObj when invoked with arguments args and keyword arguments kwargs. If a different type of exception is raised, it will not be caught, and the test case will be deemed to have suffered an error, exactly as for an unexpected exception. If called with callableObj omitted or None, will return a context object used like this:: with self.assertRaises(SomeException): do_something() The context manager keeps a reference to the exception as the 'exception' attribute. This allows you to inspect the exception after the assertion:: with self.assertRaises(SomeException) as cm: do_something() the_exception = cm.exception self.assertEqual(the_exception.error_code, 3) """ context = _AssertRaisesContext(excClass, self) if callableObj is None: return context with context: callableObj(*args, **kwargs) def _getAssertEqualityFunc(self, first, second): """Get a detailed comparison function for the types of the two args. Returns: A callable accepting (first, second, msg=None) that will raise a failure exception if first != second with a useful human readable error message for those types. """ # # NOTE(gregory.p.smith): I considered isinstance(first, type(second)) # and vice versa. I opted for the conservative approach in case # subclasses are not intended to be compared in detail to their super # class instances using a type equality func. This means testing # subtypes won't automagically use the detailed comparison. Callers # should use their type specific assertSpamEqual method to compare # subclasses if the detailed comparison is desired and appropriate. # See the discussion in http://bugs.python.org/issue2578. # if type(first) is type(second): asserter = self._type_equality_funcs.get(type(first)) if asserter is not None: if isinstance(asserter, basestring): asserter = getattr(self, asserter) return asserter return self._baseAssertEqual def _baseAssertEqual(self, first, second, msg=None): """The default assertEqual implementation, not type specific.""" if not first == second: standardMsg = '%s != %s' % (safe_repr(first), safe_repr(second)) msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) def assertEqual(self, first, second, msg=None): """Fail if the two objects are unequal as determined by the '==' operator. """ assertion_func = self._getAssertEqualityFunc(first, second) assertion_func(first, second, msg=msg) def assertNotEqual(self, first, second, msg=None): """Fail if the two objects are equal as determined by the '!=' operator. """ if not first != second: msg = self._formatMessage(msg, '%s == %s' % (safe_repr(first), safe_repr(second))) raise self.failureException(msg) def assertAlmostEqual(self, first, second, places=None, msg=None, delta=None): """Fail if the two objects are unequal as determined by their difference rounded to the given number of decimal places (default 7) and comparing to zero, or by comparing that the between the two objects is more than the given delta. Note that decimal places (from zero) are usually not the same as significant digits (measured from the most signficant digit). If the two objects compare equal then they will automatically compare almost equal. """ if first == second: # shortcut return if delta is not None and places is not None: raise TypeError("specify delta or places not both") if delta is not None: if abs(first - second) <= delta: return standardMsg = '%s != %s within %s delta' % (safe_repr(first), safe_repr(second), safe_repr(delta)) else: if places is None: places = 7 if round(abs(second-first), places) == 0: return standardMsg = '%s != %s within %r places' % (safe_repr(first), safe_repr(second), places) msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) def assertNotAlmostEqual(self, first, second, places=None, msg=None, delta=None): """Fail if the two objects are equal as determined by their difference rounded to the given number of decimal places (default 7) and comparing to zero, or by comparing that the between the two objects is less than the given delta. Note that decimal places (from zero) are usually not the same as significant digits (measured from the most signficant digit). Objects that are equal automatically fail. """ if delta is not None and places is not None: raise TypeError("specify delta or places not both") if delta is not None: if not (first == second) and abs(first - second) > delta: return standardMsg = '%s == %s within %s delta' % (safe_repr(first), safe_repr(second), safe_repr(delta)) else: if places is None: places = 7 if not (first == second) and round(abs(second-first), places) != 0: return standardMsg = '%s == %s within %r places' % (safe_repr(first), safe_repr(second), places) msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) # Synonyms for assertion methods # The plurals are undocumented. Keep them that way to discourage use. # Do not add more. Do not remove. # Going through a deprecation cycle on these would annoy many people. assertEquals = assertEqual assertNotEquals = assertNotEqual assertAlmostEquals = assertAlmostEqual assertNotAlmostEquals = assertNotAlmostEqual assert_ = assertTrue # These fail* assertion method names are pending deprecation and will # be a DeprecationWarning in 3.2; http://bugs.python.org/issue2578 def _deprecate(original_func): def deprecated_func(*args, **kwargs): warnings.warn( # 'Please use {0} instead.'.format(original_func.__name__), 'Please use %s instead.' % (original_func.__name__), PendingDeprecationWarning, 2) return original_func(*args, **kwargs) return deprecated_func failUnlessEqual = _deprecate(assertEqual) failIfEqual = _deprecate(assertNotEqual) failUnlessAlmostEqual = _deprecate(assertAlmostEqual) failIfAlmostEqual = _deprecate(assertNotAlmostEqual) failUnless = _deprecate(assertTrue) failUnlessRaises = _deprecate(assertRaises) failIf = _deprecate(assertFalse) def assertSequenceEqual(self, seq1, seq2, msg=None, seq_type=None): """An equality assertion for ordered sequences (like lists and tuples). For the purposes of this function, a valid ordered sequence type is one which can be indexed, has a length, and has an equality operator. Args: seq1: The first sequence to compare. seq2: The second sequence to compare. seq_type: The expected datatype of the sequences, or None if no datatype should be enforced. msg: Optional message to use on failure instead of a list of differences. """ if seq_type is not None: seq_type_name = seq_type.__name__ if not isinstance(seq1, seq_type): raise self.failureException('First sequence is not a %s: %s' % (seq_type_name, safe_repr(seq1))) if not isinstance(seq2, seq_type): raise self.failureException('Second sequence is not a %s: %s' % (seq_type_name, safe_repr(seq2))) else: seq_type_name = "sequence" differing = None try: len1 = len(seq1) except (TypeError, NotImplementedError): differing = 'First %s has no length. Non-sequence?' % ( seq_type_name) if differing is None: try: len2 = len(seq2) except (TypeError, NotImplementedError): differing = 'Second %s has no length. Non-sequence?' % ( seq_type_name) if differing is None: if seq1 == seq2: return seq1_repr = safe_repr(seq1) seq2_repr = safe_repr(seq2) if len(seq1_repr) > 30: seq1_repr = seq1_repr[:30] + '...' if len(seq2_repr) > 30: seq2_repr = seq2_repr[:30] + '...' elements = (seq_type_name.capitalize(), seq1_repr, seq2_repr) differing = '%ss differ: %s != %s\n' % elements for i in xrange(min(len1, len2)): try: item1 = seq1[i] except (TypeError, IndexError, NotImplementedError): differing += ('\nUnable to index element %d of first %s\n' % (i, seq_type_name)) break try: item2 = seq2[i] except (TypeError, IndexError, NotImplementedError): differing += ('\nUnable to index element %d of second %s\n' % (i, seq_type_name)) break if item1 != item2: differing += ('\nFirst differing element %d:\n%s\n%s\n' % (i, safe_repr(item1), safe_repr(item2))) break else: if (len1 == len2 and seq_type is None and type(seq1) != type(seq2)): # The sequences are the same, but have differing types. return if len1 > len2: differing += ('\nFirst %s contains %d additional ' 'elements.\n' % (seq_type_name, len1 - len2)) try: differing += ('First extra element %d:\n%s\n' % (len2, safe_repr(seq1[len2]))) except (TypeError, IndexError, NotImplementedError): differing += ('Unable to index element %d ' 'of first %s\n' % (len2, seq_type_name)) elif len1 < len2: differing += ('\nSecond %s contains %d additional ' 'elements.\n' % (seq_type_name, len2 - len1)) try: differing += ('First extra element %d:\n%s\n' % (len1, safe_repr(seq2[len1]))) except (TypeError, IndexError, NotImplementedError): differing += ('Unable to index element %d ' 'of second %s\n' % (len1, seq_type_name)) standardMsg = differing diffMsg = '\n' + '\n'.join( difflib.ndiff(pprint.pformat(seq1).splitlines(), pprint.pformat(seq2).splitlines())) standardMsg = self._truncateMessage(standardMsg, diffMsg) msg = self._formatMessage(msg, standardMsg) self.fail(msg) def _truncateMessage(self, message, diff): max_diff = self.maxDiff if max_diff is None or len(diff) <= max_diff: return message + diff return message + (DIFF_OMITTED % len(diff)) def assertListEqual(self, list1, list2, msg=None): """A list-specific equality assertion. Args: list1: The first list to compare. list2: The second list to compare. msg: Optional message to use on failure instead of a list of differences. """ self.assertSequenceEqual(list1, list2, msg, seq_type=list) def assertTupleEqual(self, tuple1, tuple2, msg=None): """A tuple-specific equality assertion. Args: tuple1: The first tuple to compare. tuple2: The second tuple to compare. msg: Optional message to use on failure instead of a list of differences. """ self.assertSequenceEqual(tuple1, tuple2, msg, seq_type=tuple) def assertSetEqual(self, set1, set2, msg=None): """A set-specific equality assertion. Args: set1: The first set to compare. set2: The second set to compare. msg: Optional message to use on failure instead of a list of differences. assertSetEqual uses ducktyping to support different types of sets, and is optimized for sets specifically (parameters must support a difference method). """ try: difference1 = set1.difference(set2) except TypeError, e: self.fail('invalid type when attempting set difference: %s' % e) except AttributeError, e: self.fail('first argument does not support set difference: %s' % e) try: difference2 = set2.difference(set1) except TypeError, e: self.fail('invalid type when attempting set difference: %s' % e) except AttributeError, e: self.fail('second argument does not support set difference: %s' % e) if not (difference1 or difference2): return lines = [] if difference1: lines.append('Items in the first set but not the second:') for item in difference1: lines.append(repr(item)) if difference2: lines.append('Items in the second set but not the first:') for item in difference2: lines.append(repr(item)) standardMsg = '\n'.join(lines) self.fail(self._formatMessage(msg, standardMsg)) def assertIn(self, member, container, msg=None): """Just like self.assertTrue(a in b), but with a nicer default message.""" if member not in container: standardMsg = '%s not found in %s' % (safe_repr(member), safe_repr(container)) self.fail(self._formatMessage(msg, standardMsg)) def assertNotIn(self, member, container, msg=None): """Just like self.assertTrue(a not in b), but with a nicer default message.""" if member in container: standardMsg = '%s unexpectedly found in %s' % (safe_repr(member), safe_repr(container)) self.fail(self._formatMessage(msg, standardMsg)) def assertIs(self, expr1, expr2, msg=None): """Just like self.assertTrue(a is b), but with a nicer default message.""" if expr1 is not expr2: standardMsg = '%s is not %s' % (safe_repr(expr1), safe_repr(expr2)) self.fail(self._formatMessage(msg, standardMsg)) def assertIsNot(self, expr1, expr2, msg=None): """Just like self.assertTrue(a is not b), but with a nicer default message.""" if expr1 is expr2: standardMsg = 'unexpectedly identical: %s' % (safe_repr(expr1),) self.fail(self._formatMessage(msg, standardMsg)) def assertDictEqual(self, d1, d2, msg=None): self.assertIsInstance(d1, dict, 'First argument is not a dictionary') self.assertIsInstance(d2, dict, 'Second argument is not a dictionary') if d1 != d2: standardMsg = '%s != %s' % (safe_repr(d1, True), safe_repr(d2, True)) diff = ('\n' + '\n'.join(difflib.ndiff( pprint.pformat(d1).splitlines(), pprint.pformat(d2).splitlines()))) standardMsg = self._truncateMessage(standardMsg, diff) self.fail(self._formatMessage(msg, standardMsg)) def assertDictContainsSubset(self, expected, actual, msg=None): """Checks whether actual is a superset of expected.""" missing = [] mismatched = [] for key, value in expected.iteritems(): if key not in actual: missing.append(key) elif value != actual[key]: mismatched.append('%s, expected: %s, actual: %s' % (safe_repr(key), safe_repr(value), safe_repr(actual[key]))) if not (missing or mismatched): return standardMsg = '' if missing: standardMsg = 'Missing: %s' % ','.join(safe_repr(m) for m in missing) if mismatched: if standardMsg: standardMsg += '; ' standardMsg += 'Mismatched values: %s' % ','.join(mismatched) self.fail(self._formatMessage(msg, standardMsg)) def assertItemsEqual(self, expected_seq, actual_seq, msg=None): """An unordered sequence specific comparison. It asserts that actual_seq and expected_seq have the same element counts. Equivalent to:: self.assertEqual(Counter(iter(actual_seq)), Counter(iter(expected_seq))) Asserts that each element has the same count in both sequences. Example: - [0, 1, 1] and [1, 0, 1] compare equal. - [0, 0, 1] and [0, 1] compare unequal. """ first_seq, second_seq = list(expected_seq), list(actual_seq) with warnings.catch_warnings(): if sys.py3kwarning: # Silence Py3k warning raised during the sorting for _msg in ["(code|dict|type) inequality comparisons", "builtin_function_or_method order comparisons", "comparing unequal types"]: warnings.filterwarnings("ignore", _msg, DeprecationWarning) try: first = collections.Counter(first_seq) second = collections.Counter(second_seq) except TypeError: # Handle case with unhashable elements differences = _count_diff_all_purpose(first_seq, second_seq) else: if first == second: return differences = _count_diff_hashable(first_seq, second_seq) if differences: standardMsg = 'Element counts were not equal:\n' lines = ['First has %d, Second has %d: %r' % diff for diff in differences] diffMsg = '\n'.join(lines) standardMsg = self._truncateMessage(standardMsg, diffMsg) msg = self._formatMessage(msg, standardMsg) self.fail(msg) def assertMultiLineEqual(self, first, second, msg=None): """Assert that two multi-line strings are equal.""" self.assertIsInstance(first, basestring, 'First argument is not a string') self.assertIsInstance(second, basestring, 'Second argument is not a string') if first != second: # don't use difflib if the strings are too long if (len(first) > self._diffThreshold or len(second) > self._diffThreshold): self._baseAssertEqual(first, second, msg) firstlines = first.splitlines(True) secondlines = second.splitlines(True) if len(firstlines) == 1 and first.strip('\r\n') == first: firstlines = [first + '\n'] secondlines = [second + '\n'] standardMsg = '%s != %s' % (safe_repr(first, True), safe_repr(second, True)) diff = '\n' + ''.join(difflib.ndiff(firstlines, secondlines)) standardMsg = self._truncateMessage(standardMsg, diff) self.fail(self._formatMessage(msg, standardMsg)) def assertLess(self, a, b, msg=None): """Just like self.assertTrue(a < b), but with a nicer default message.""" if not a < b: standardMsg = '%s not less than %s' % (safe_repr(a), safe_repr(b)) self.fail(self._formatMessage(msg, standardMsg)) def assertLessEqual(self, a, b, msg=None): """Just like self.assertTrue(a <= b), but with a nicer default message.""" if not a <= b: standardMsg = '%s not less than or equal to %s' % (safe_repr(a), safe_repr(b)) self.fail(self._formatMessage(msg, standardMsg)) def assertGreater(self, a, b, msg=None): """Just like self.assertTrue(a > b), but with a nicer default message.""" if not a > b: standardMsg = '%s not greater than %s' % (safe_repr(a), safe_repr(b)) self.fail(self._formatMessage(msg, standardMsg)) def assertGreaterEqual(self, a, b, msg=None): """Just like self.assertTrue(a >= b), but with a nicer default message.""" if not a >= b: standardMsg = '%s not greater than or equal to %s' % (safe_repr(a), safe_repr(b)) self.fail(self._formatMessage(msg, standardMsg)) def assertIsNone(self, obj, msg=None): """Same as self.assertTrue(obj is None), with a nicer default message.""" if obj is not None: standardMsg = '%s is not None' % (safe_repr(obj),) self.fail(self._formatMessage(msg, standardMsg)) def assertIsNotNone(self, obj, msg=None): """Included for symmetry with assertIsNone.""" if obj is None: standardMsg = 'unexpectedly None' self.fail(self._formatMessage(msg, standardMsg)) def assertIsInstance(self, obj, cls, msg=None): """Same as self.assertTrue(isinstance(obj, cls)), with a nicer default message.""" if not isinstance(obj, cls): standardMsg = '%s is not an instance of %r' % (safe_repr(obj), cls) self.fail(self._formatMessage(msg, standardMsg)) def assertNotIsInstance(self, obj, cls, msg=None): """Included for symmetry with assertIsInstance.""" if isinstance(obj, cls): standardMsg = '%s is an instance of %r' % (safe_repr(obj), cls) self.fail(self._formatMessage(msg, standardMsg)) def assertRaisesRegexp(self, expected_exception, expected_regexp, callable_obj=None, *args, **kwargs): """Asserts that the message in a raised exception matches a regexp. Args: expected_exception: Exception class expected to be raised. expected_regexp: Regexp (re pattern object or string) expected to be found in error message. callable_obj: Function to be called. args: Extra args. kwargs: Extra kwargs. """ if expected_regexp is not None: expected_regexp = re.compile(expected_regexp) context = _AssertRaisesContext(expected_exception, self, expected_regexp) if callable_obj is None: return context with context: callable_obj(*args, **kwargs) def assertRegexpMatches(self, text, expected_regexp, msg=None): """Fail the test unless the text matches the regular expression.""" if isinstance(expected_regexp, basestring): expected_regexp = re.compile(expected_regexp) if not expected_regexp.search(text): msg = msg or "Regexp didn't match" msg = '%s: %r not found in %r' % (msg, expected_regexp.pattern, text) raise self.failureException(msg) def assertNotRegexpMatches(self, text, unexpected_regexp, msg=None): """Fail the test if the text matches the regular expression.""" if isinstance(unexpected_regexp, basestring): unexpected_regexp = re.compile(unexpected_regexp) match = unexpected_regexp.search(text) if match: msg = msg or "Regexp matched" msg = '%s: %r matches %r in %r' % (msg, text[match.start():match.end()], unexpected_regexp.pattern, text) raise self.failureException(msg) class FunctionTestCase(TestCase): """A test case that wraps a test function. This is useful for slipping pre-existing test functions into the unittest framework. Optionally, set-up and tidy-up functions can be supplied. As with TestCase, the tidy-up ('tearDown') function will always be called if the set-up ('setUp') function ran successfully. """ def __init__(self, testFunc, setUp=None, tearDown=None, description=None): super(FunctionTestCase, self).__init__() self._setUpFunc = setUp self._tearDownFunc = tearDown self._testFunc = testFunc self._description = description def setUp(self): if self._setUpFunc is not None: self._setUpFunc() def tearDown(self): if self._tearDownFunc is not None: self._tearDownFunc() def runTest(self): self._testFunc() def id(self): return self._testFunc.__name__ def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented return self._setUpFunc == other._setUpFunc and \ self._tearDownFunc == other._tearDownFunc and \ self._testFunc == other._testFunc and \ self._description == other._description def __ne__(self, other): return not self == other def __hash__(self): return hash((type(self), self._setUpFunc, self._tearDownFunc, self._testFunc, self._description)) def __str__(self): return "%s (%s)" % (strclass(self.__class__), self._testFunc.__name__) def __repr__(self): return "<%s tec=%s>" % (strclass(self.__class__), self._testFunc) def shortDescription(self): if self._description is not None: return self._description # doc = self._testFunc.__doc__ return doc and doc.split("\n")[0].strip() or None ================================================ FILE: third_party/stdlib/unittest_loader.py ================================================ """Loading unittests.""" import os import re import sys import traceback import types # from functools import cmp_to_key as _CmpToKey # from fnmatch import fnmatch import functools import fnmatch as _fnmatch _CmpToKey = functools.cmp_to_key fnmatch = _fnmatch.fnmatch # from . import case, suite import unittest_case as case import unittest_suite as suite __unittest = True # what about .pyc or .pyo (etc) # we would need to avoid loading the same tests multiple times # from '.py', '.pyc' *and* '.pyo' VALID_MODULE_NAME = re.compile(r'[_a-z]\w*\.py$', re.IGNORECASE) def _make_failed_import_test(name, suiteClass): message = 'Failed to import test module: %s\n%s' % (name, traceback.format_exc()) return _make_failed_test('ModuleImportFailure', name, ImportError(message), suiteClass) def _make_failed_load_tests(name, exception, suiteClass): return _make_failed_test('LoadTestsFailure', name, exception, suiteClass) def _make_failed_test(classname, methodname, exception, suiteClass): def testFailure(self): raise exception attrs = {methodname: testFailure} TestClass = type(classname, (case.TestCase,), attrs) return suiteClass((TestClass(methodname),)) class TestLoader(object): """ This class is responsible for loading tests according to various criteria and returning them wrapped in a TestSuite """ testMethodPrefix = 'test' sortTestMethodsUsing = cmp suiteClass = suite.TestSuite _top_level_dir = None def loadTestsFromTestCase(self, testCaseClass): """Return a suite of all tests cases contained in testCaseClass""" if issubclass(testCaseClass, suite.TestSuite): raise TypeError("Test cases should not be derived from TestSuite." \ " Maybe you meant to derive from TestCase?") testCaseNames = self.getTestCaseNames(testCaseClass) if not testCaseNames and hasattr(testCaseClass, 'runTest'): testCaseNames = ['runTest'] loaded_suite = self.suiteClass(map(testCaseClass, testCaseNames)) return loaded_suite def loadTestsFromModule(self, module, use_load_tests=True): """Return a suite of all tests cases contained in the given module""" tests = [] for name in dir(module): obj = getattr(module, name) if isinstance(obj, type) and issubclass(obj, case.TestCase): tests.append(self.loadTestsFromTestCase(obj)) load_tests = getattr(module, 'load_tests', None) tests = self.suiteClass(tests) if use_load_tests and load_tests is not None: try: return load_tests(self, tests, None) except Exception, e: return _make_failed_load_tests(module.__name__, e, self.suiteClass) return tests def loadTestsFromName(self, name, module=None): """Return a suite of all tests cases given a string specifier. The name may resolve either to a module, a test case class, a test method within a test case class, or a callable object which returns a TestCase or TestSuite instance. The method optionally resolves the names relative to a given module. """ parts = name.split('.') if module is None: parts_copy = parts[:] while parts_copy: try: module = __import__('.'.join(parts_copy)) break except ImportError: del parts_copy[-1] if not parts_copy: raise parts = parts[1:] obj = module for part in parts: parent, obj = obj, getattr(obj, part) if isinstance(obj, types.ModuleType): return self.loadTestsFromModule(obj) elif isinstance(obj, type) and issubclass(obj, case.TestCase): return self.loadTestsFromTestCase(obj) elif (isinstance(obj, types.UnboundMethodType) and isinstance(parent, type) and issubclass(parent, case.TestCase)): name = parts[-1] inst = parent(name) return self.suiteClass([inst]) elif isinstance(obj, suite.TestSuite): return obj elif hasattr(obj, '__call__'): test = obj() if isinstance(test, suite.TestSuite): return test elif isinstance(test, case.TestCase): return self.suiteClass([test]) else: raise TypeError("calling %s returned %s, not a test" % (obj, test)) else: raise TypeError("don't know how to make test from: %s" % obj) def loadTestsFromNames(self, names, module=None): """Return a suite of all tests cases found using the given sequence of string specifiers. See 'loadTestsFromName()'. """ suites = [self.loadTestsFromName(name, module) for name in names] return self.suiteClass(suites) def getTestCaseNames(self, testCaseClass): """Return a sorted sequence of method names found within testCaseClass """ def isTestMethod(attrname, testCaseClass=testCaseClass, prefix=self.testMethodPrefix): return attrname.startswith(prefix) and \ hasattr(getattr(testCaseClass, attrname), '__call__') # testFnNames = filter(isTestMethod, dir(testCaseClass)) testFnNames = [x for x in dir(testCaseClass) if isTestMethod(x)] if self.sortTestMethodsUsing: testFnNames.sort(key=_CmpToKey(self.sortTestMethodsUsing)) return testFnNames def discover(self, start_dir, pattern='test*.py', top_level_dir=None): """Find and return all test modules from the specified start directory, recursing into subdirectories to find them. Only test files that match the pattern will be loaded. (Using shell style pattern matching.) All test modules must be importable from the top level of the project. If the start directory is not the top level directory then the top level directory must be specified separately. If a test package name (directory with '__init__.py') matches the pattern then the package will be checked for a 'load_tests' function. If this exists then it will be called with loader, tests, pattern. If load_tests exists then discovery does *not* recurse into the package, load_tests is responsible for loading all tests in the package. The pattern is deliberately not stored as a loader attribute so that packages can continue discovery themselves. top_level_dir is stored so load_tests does not need to pass this argument in to loader.discover(). """ set_implicit_top = False if top_level_dir is None and self._top_level_dir is not None: # make top_level_dir optional if called from load_tests in a package top_level_dir = self._top_level_dir elif top_level_dir is None: set_implicit_top = True top_level_dir = start_dir top_level_dir = os.path.abspath(top_level_dir) if not top_level_dir in sys.path: # all test modules must be importable from the top level directory # should we *unconditionally* put the start directory in first # in sys.path to minimise likelihood of conflicts between installed # modules and development versions? sys.path.insert(0, top_level_dir) self._top_level_dir = top_level_dir is_not_importable = False if os.path.isdir(os.path.abspath(start_dir)): start_dir = os.path.abspath(start_dir) if start_dir != top_level_dir: is_not_importable = not os.path.isfile(os.path.join(start_dir, '__init__.py')) else: # support for discovery from dotted module names try: __import__(start_dir) except ImportError: is_not_importable = True else: the_module = sys.modules[start_dir] top_part = start_dir.split('.')[0] start_dir = os.path.abspath(os.path.dirname((the_module.__file__))) if set_implicit_top: self._top_level_dir = self._get_directory_containing_module(top_part) sys.path.remove(top_level_dir) if is_not_importable: raise ImportError('Start directory is not importable: %r' % start_dir) tests = list(self._find_tests(start_dir, pattern)) return self.suiteClass(tests) def _get_directory_containing_module(self, module_name): module = sys.modules[module_name] full_path = os.path.abspath(module.__file__) if os.path.basename(full_path).lower().startswith('__init__.py'): return os.path.dirname(os.path.dirname(full_path)) else: # here we have been given a module rather than a package - so # all we can do is search the *same* directory the module is in # should an exception be raised instead return os.path.dirname(full_path) def _get_name_from_path(self, path): path = os.path.splitext(os.path.normpath(path))[0] _relpath = os.path.relpath(path, self._top_level_dir) assert not os.path.isabs(_relpath), "Path must be within the project" assert not _relpath.startswith('..'), "Path must be within the project" name = _relpath.replace(os.path.sep, '.') return name def _get_module_from_name(self, name): __import__(name) return sys.modules[name] def _match_path(self, path, full_path, pattern): # override this method to use alternative matching strategy return fnmatch(path, pattern) def _find_tests(self, start_dir, pattern): """Used by discovery. Yields test suites it loads.""" paths = os.listdir(start_dir) for path in paths: full_path = os.path.join(start_dir, path) if os.path.isfile(full_path): if not VALID_MODULE_NAME.match(path): # valid Python identifiers only continue if not self._match_path(path, full_path, pattern): continue # if the test file matches, load it name = self._get_name_from_path(full_path) try: module = self._get_module_from_name(name) except: yield _make_failed_import_test(name, self.suiteClass) else: mod_file = os.path.abspath(getattr(module, '__file__', full_path)) realpath = os.path.splitext(os.path.realpath(mod_file))[0] fullpath_noext = os.path.splitext(os.path.realpath(full_path))[0] if realpath.lower() != fullpath_noext.lower(): module_dir = os.path.dirname(realpath) mod_name = os.path.splitext(os.path.basename(full_path))[0] expected_dir = os.path.dirname(full_path) msg = ("%r module incorrectly imported from %r. Expected %r. " "Is this module globally installed?") raise ImportError(msg % (mod_name, module_dir, expected_dir)) yield self.loadTestsFromModule(module) elif os.path.isdir(full_path): if not os.path.isfile(os.path.join(full_path, '__init__.py')): continue load_tests = None tests = None if fnmatch(path, pattern): # only check load_tests if the package directory itself matches the filter name = self._get_name_from_path(full_path) package = self._get_module_from_name(name) load_tests = getattr(package, 'load_tests', None) tests = self.loadTestsFromModule(package, use_load_tests=False) if load_tests is None: if tests is not None: # tests loaded from package file yield tests # recurse into the package for test in self._find_tests(full_path, pattern): yield test else: try: yield load_tests(self, tests, pattern) except Exception, e: yield _make_failed_load_tests(package.__name__, e, self.suiteClass) defaultTestLoader = TestLoader() def _makeLoader(prefix, sortUsing, suiteClass=None): loader = TestLoader() loader.sortTestMethodsUsing = sortUsing loader.testMethodPrefix = prefix if suiteClass: loader.suiteClass = suiteClass return loader def getTestCaseNames(testCaseClass, prefix, sortUsing=cmp): return _makeLoader(prefix, sortUsing).getTestCaseNames(testCaseClass) def makeSuite(testCaseClass, prefix='test', sortUsing=cmp, suiteClass=suite.TestSuite): return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromTestCase(testCaseClass) def findTestCases(module, prefix='test', sortUsing=cmp, suiteClass=suite.TestSuite): return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromModule(module) ================================================ FILE: third_party/stdlib/unittest_result.py ================================================ """Test result object""" import os import sys import traceback # from StringIO import StringIO import StringIO as _StringIO StringIO = _StringIO.StringIO # from . import util # from functools import wraps import unittest_util as util import functools wraps = functools.wraps __unittest = True def failfast(method): # @wraps(method) def inner(self, *args, **kw): if getattr(self, 'failfast', False): self.stop() return method(self, *args, **kw) inner = wraps(method)(inner) return inner STDOUT_LINE = '\nStdout:\n%s' STDERR_LINE = '\nStderr:\n%s' class TestResult(object): """Holder for test result information. Test results are automatically managed by the TestCase and TestSuite classes, and do not need to be explicitly manipulated by writers of tests. Each instance holds the total number of tests run, and collections of failures and errors that occurred among those test runs. The collections contain tuples of (testcase, exceptioninfo), where exceptioninfo is the formatted traceback of the error that occurred. """ _previousTestClass = None _testRunEntered = False _moduleSetUpFailed = False def __init__(self, stream=None, descriptions=None, verbosity=None): self.failfast = False self.failures = [] self.errors = [] self.testsRun = 0 self.skipped = [] self.expectedFailures = [] self.unexpectedSuccesses = [] self.shouldStop = False self.buffer = False self._stdout_buffer = None self._stderr_buffer = None self._original_stdout = sys.stdout self._original_stderr = sys.stderr self._mirrorOutput = False def printErrors(self): "Called by TestRunner after test run" def startTest(self, test): "Called when the given test is about to be run" self.testsRun += 1 self._mirrorOutput = False self._setupStdout() def _setupStdout(self): if self.buffer: if self._stderr_buffer is None: self._stderr_buffer = StringIO() self._stdout_buffer = StringIO() sys.stdout = self._stdout_buffer sys.stderr = self._stderr_buffer def startTestRun(self): """Called once before any tests are executed. See startTest for a method called before each test. """ def stopTest(self, test): """Called when the given test has been run""" self._restoreStdout() self._mirrorOutput = False def _restoreStdout(self): if self.buffer: if self._mirrorOutput: output = sys.stdout.getvalue() error = sys.stderr.getvalue() if output: if not output.endswith('\n'): output += '\n' self._original_stdout.write(STDOUT_LINE % output) if error: if not error.endswith('\n'): error += '\n' self._original_stderr.write(STDERR_LINE % error) sys.stdout = self._original_stdout sys.stderr = self._original_stderr self._stdout_buffer.seek(0) self._stdout_buffer.truncate() self._stderr_buffer.seek(0) self._stderr_buffer.truncate() def stopTestRun(self): """Called once after all tests are executed. See stopTest for a method called after each test. """ # @failfast def addError(self, test, err): """Called when an error has occurred. 'err' is a tuple of values as returned by sys.exc_info(). """ self.errors.append((test, self._exc_info_to_string(err, test))) self._mirrorOutput = True addError = failfast(addError) # @failfast def addFailure(self, test, err): """Called when an error has occurred. 'err' is a tuple of values as returned by sys.exc_info().""" self.failures.append((test, self._exc_info_to_string(err, test))) self._mirrorOutput = True addFailure = failfast(addFailure) def addSuccess(self, test): "Called when a test has completed successfully" pass def addSkip(self, test, reason): """Called when a test is skipped.""" self.skipped.append((test, reason)) def addExpectedFailure(self, test, err): """Called when an expected failure/error occurred.""" self.expectedFailures.append( (test, self._exc_info_to_string(err, test))) # @failfast def addUnexpectedSuccess(self, test): """Called when a test was expected to fail, but succeed.""" self.unexpectedSuccesses.append(test) addUnexpectedSuccess = failfast(addUnexpectedSuccess) def wasSuccessful(self): "Tells whether or not this result was a success" return len(self.failures) == len(self.errors) == 0 def stop(self): "Indicates that the tests should be aborted" self.shouldStop = True def _exc_info_to_string(self, err, test): """Converts a sys.exc_info()-style tuple of values into a string.""" exctype, value, tb = err # Skip test runner traceback levels while tb and self._is_relevant_tb_level(tb): tb = tb.tb_next if exctype is test.failureException: # Skip assert*() traceback levels length = self._count_relevant_tb_levels(tb) msgLines = traceback.format_exception(exctype, value, tb, length) else: msgLines = traceback.format_exception(exctype, value, tb) if self.buffer: output = sys.stdout.getvalue() error = sys.stderr.getvalue() if output: if not output.endswith('\n'): output += '\n' msgLines.append(STDOUT_LINE % output) if error: if not error.endswith('\n'): error += '\n' msgLines.append(STDERR_LINE % error) return ''.join(msgLines) def _is_relevant_tb_level(self, tb): return '__unittest' in tb.tb_frame.f_globals def _count_relevant_tb_levels(self, tb): length = 0 while tb and not self._is_relevant_tb_level(tb): length += 1 tb = tb.tb_next return length def __repr__(self): return ("<%s run=%i errors=%i failures=%i>" % (util.strclass(self.__class__), self.testsRun, len(self.errors), len(self.failures))) ================================================ FILE: third_party/stdlib/unittest_runner.py ================================================ """Running tests""" import sys import time # from . import result # from .signals import registerResult import unittest_result as result import unittest_signals registerResult = unittest_signals.registerResult __unittest = True class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" def __init__(self, stream): self.stream = stream def __getattr__(self, attr): if attr in ('stream', '__getstate__'): raise AttributeError(attr) return getattr(self.stream,attr) def writeln(self, arg=None): if arg: self.write(arg) self.write('\n') # text-mode streams translate to \r\n if needed def write(self, arg): self.stream.write(arg) def flush(self): pass class TextTestResult(result.TestResult): """A test result class that can print formatted text results to a stream. Used by TextTestRunner. """ separator1 = '=' * 70 separator2 = '-' * 70 def __init__(self, stream, descriptions, verbosity): super(TextTestResult, self).__init__(stream, descriptions, verbosity) self.stream = stream self.showAll = verbosity > 1 self.dots = verbosity == 1 self.descriptions = descriptions def getDescription(self, test): doc_first_line = test.shortDescription() if self.descriptions and doc_first_line: return '\n'.join((str(test), doc_first_line)) else: return str(test) def startTest(self, test): super(TextTestResult, self).startTest(test) if self.showAll: self.stream.write(self.getDescription(test)) self.stream.write(" ... ") self.stream.flush() def addSuccess(self, test): super(TextTestResult, self).addSuccess(test) if self.showAll: self.stream.writeln("ok") elif self.dots: self.stream.write('.') self.stream.flush() def addError(self, test, err): super(TextTestResult, self).addError(test, err) if self.showAll: self.stream.writeln("ERROR") elif self.dots: self.stream.write('E') self.stream.flush() def addFailure(self, test, err): super(TextTestResult, self).addFailure(test, err) if self.showAll: self.stream.writeln("FAIL") elif self.dots: self.stream.write('F') self.stream.flush() def addSkip(self, test, reason): super(TextTestResult, self).addSkip(test, reason) if self.showAll: # self.stream.writeln("skipped {0!r}".format(reason)) self.stream.writeln("skipped %r" % (reason)) elif self.dots: self.stream.write("s") self.stream.flush() def addExpectedFailure(self, test, err): super(TextTestResult, self).addExpectedFailure(test, err) if self.showAll: self.stream.writeln("expected failure") elif self.dots: self.stream.write("x") self.stream.flush() def addUnexpectedSuccess(self, test): super(TextTestResult, self).addUnexpectedSuccess(test) if self.showAll: self.stream.writeln("unexpected success") elif self.dots: self.stream.write("u") self.stream.flush() def printErrors(self): if self.dots or self.showAll: self.stream.writeln() self.printErrorList('ERROR', self.errors) self.printErrorList('FAIL', self.failures) def printErrorList(self, flavour, errors): for test, err in errors: self.stream.writeln(self.separator1) self.stream.writeln("%s: %s" % (flavour,self.getDescription(test))) self.stream.writeln(self.separator2) self.stream.writeln("%s" % err) class TextTestRunner(object): """A test runner class that displays results in textual form. It prints out the names of tests as they are run, errors as they occur, and a summary of the results at the end of the test run. """ resultclass = TextTestResult def __init__(self, stream=sys.stderr, descriptions=True, verbosity=1, failfast=False, buffer=False, resultclass=None): self.stream = _WritelnDecorator(stream) self.descriptions = descriptions self.verbosity = verbosity self.failfast = failfast self.buffer = buffer if resultclass is not None: self.resultclass = resultclass def _makeResult(self): return self.resultclass(self.stream, self.descriptions, self.verbosity) def run(self, test): "Run the given test case or test suite." result = self._makeResult() registerResult(result) result.failfast = self.failfast result.buffer = self.buffer startTime = time.time() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() try: test(result) finally: stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() stopTime = time.time() timeTaken = stopTime - startTime result.printErrors() if hasattr(result, 'separator2'): self.stream.writeln(result.separator2) run = result.testsRun # self.stream.writeln("Ran %d test%s in %.3fs" % self.stream.writeln("Ran %d test%s in %fs" % (run, run != 1 and "s" or "", timeTaken)) self.stream.writeln() expectedFails = unexpectedSuccesses = skipped = 0 try: results = map(len, (result.expectedFailures, result.unexpectedSuccesses, result.skipped)) except AttributeError: pass else: expectedFails, unexpectedSuccesses, skipped = results infos = [] if not result.wasSuccessful(): self.stream.write("FAILED") failed, errored = map(len, (result.failures, result.errors)) if failed: infos.append("failures=%d" % failed) if errored: infos.append("errors=%d" % errored) else: self.stream.write("OK") if skipped: infos.append("skipped=%d" % skipped) if expectedFails: infos.append("expected failures=%d" % expectedFails) if unexpectedSuccesses: infos.append("unexpected successes=%d" % unexpectedSuccesses) if infos: self.stream.writeln(" (%s)" % (", ".join(infos),)) else: self.stream.write("\n") return result ================================================ FILE: third_party/stdlib/unittest_signals.py ================================================ # TODO: support signal # import signal # import weakref # from functools import wraps import functools wraps = functools.wraps __unittest = True class _InterruptHandler(object): pass # def __init__(self, default_handler): # self.called = False # self.original_handler = default_handler # if isinstance(default_handler, int): # print 'signal not supported yet' # if default_handler == signal.SIG_DFL: # # Pretend it's signal.default_int_handler instead. # default_handler = signal.default_int_handler # elif default_handler == signal.SIG_IGN: # # Not quite the same thing as SIG_IGN, but the closest we # # can make it: do nothing. # def default_handler(unused_signum, unused_frame): # pass # else: # raise TypeError("expected SIGINT signal handler to be " # "signal.SIG_IGN, signal.SIG_DFL, or a " # "callable object") # self.default_handler = default_handler # def __call__(self, signum, frame): # installed_handler = signal.getsignal(signal.SIGINT) # if installed_handler is not self: # # if we aren't the installed handler, then delegate immediately # # to the default handler # self.default_handler(signum, frame) # if self.called: # self.default_handler(signum, frame) # self.called = True # for result in _results.keys(): # result.stop() # _results = weakref.WeakKeyDictionary() _results = {} def registerResult(result): _results[result] = 1 def removeResult(result): return bool(_results.pop(result, None)) _interrupt_handler = None def installHandler(): global _interrupt_handler pass # if _interrupt_handler is None: # default_handler = signal.getsignal(signal.SIGINT) # _interrupt_handler = _InterruptHandler(default_handler) # signal.signal(signal.SIGINT, _interrupt_handler) def removeHandler(method=None): pass # if method is not None: # # @wraps(method) # def inner(*args, **kwargs): # initial = signal.getsignal(signal.SIGINT) # removeHandler() # try: # return method(*args, **kwargs) # finally: # signal.signal(signal.SIGINT, initial) # inner = wraps(method)(inner) # return inner # global _interrupt_handler # if _interrupt_handler is not None: # signal.signal(signal.SIGINT, _interrupt_handler.original_handler) ================================================ FILE: third_party/stdlib/unittest_suite.py ================================================ """TestSuite""" import sys # from . import case # from . import util import unittest_case as case import unittest_util as util __unittest = True def _call_if_exists(parent, attr): func = getattr(parent, attr, lambda: None) func() class BaseTestSuite(object): """A simple test suite that doesn't provide class or module shared fixtures. """ def __init__(self, tests=()): self._tests = [] self.addTests(tests) def __repr__(self): return "<%s tests=%s>" % (util.strclass(self.__class__), list(self)) def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented return list(self) == list(other) def __ne__(self, other): return not self == other # Can't guarantee hash invariant, so flag as unhashable __hash__ = None def __iter__(self): return iter(self._tests) def countTestCases(self): cases = 0 for test in self: cases += test.countTestCases() return cases def addTest(self, test): # sanity checks if not hasattr(test, '__call__'): # raise TypeError("{} is not callable".format(repr(test))) raise TypeError("%s is not callable" % (repr(test))) if isinstance(test, type) and issubclass(test, (case.TestCase, TestSuite)): raise TypeError("TestCases and TestSuites must be instantiated " "before passing them to addTest()") self._tests.append(test) def addTests(self, tests): if isinstance(tests, basestring): raise TypeError("tests must be an iterable of tests, not a string") for test in tests: self.addTest(test) def run(self, result): for test in self: if result.shouldStop: break test(result) return result def __call__(self, *args, **kwds): return self.run(*args, **kwds) def debug(self): """Run the tests without collecting errors in a TestResult""" for test in self: test.debug() class TestSuite(BaseTestSuite): """A test suite is a composite test consisting of a number of TestCases. For use, create an instance of TestSuite, then add test case instances. When all tests have been added, the suite can be passed to a test runner, such as TextTestRunner. It will run the individual test cases in the order in which they were added, aggregating the results. When subclassing, do not forget to call the base class constructor. """ def run(self, result, debug=False): topLevel = False if getattr(result, '_testRunEntered', False) is False: result._testRunEntered = topLevel = True for test in self: if result.shouldStop: break if _isnotsuite(test): self._tearDownPreviousClass(test, result) self._handleModuleFixture(test, result) self._handleClassSetUp(test, result) result._previousTestClass = test.__class__ if (getattr(test.__class__, '_classSetupFailed', False) or getattr(result, '_moduleSetUpFailed', False)): continue if not debug: test(result) else: test.debug() if topLevel: self._tearDownPreviousClass(None, result) self._handleModuleTearDown(result) result._testRunEntered = False return result def debug(self): """Run the tests without collecting errors in a TestResult""" debug = _DebugResult() self.run(debug, True) ################################ def _handleClassSetUp(self, test, result): previousClass = getattr(result, '_previousTestClass', None) currentClass = test.__class__ if currentClass == previousClass: return if result._moduleSetUpFailed: return if getattr(currentClass, "__unittest_skip__", False): return try: currentClass._classSetupFailed = False except TypeError: # test may actually be a function # so its class will be a builtin-type pass setUpClass = getattr(currentClass, 'setUpClass', None) if setUpClass is not None: _call_if_exists(result, '_setupStdout') try: setUpClass() except Exception as e: if isinstance(result, _DebugResult): raise currentClass._classSetupFailed = True className = util.strclass(currentClass) errorName = 'setUpClass (%s)' % className self._addClassOrModuleLevelException(result, e, errorName) finally: _call_if_exists(result, '_restoreStdout') def _get_previous_module(self, result): previousModule = None previousClass = getattr(result, '_previousTestClass', None) if previousClass is not None: previousModule = previousClass.__module__ return previousModule def _handleModuleFixture(self, test, result): previousModule = self._get_previous_module(result) currentModule = test.__class__.__module__ if currentModule == previousModule: return self._handleModuleTearDown(result) result._moduleSetUpFailed = False try: module = sys.modules[currentModule] except KeyError: return setUpModule = getattr(module, 'setUpModule', None) if setUpModule is not None: _call_if_exists(result, '_setupStdout') try: setUpModule() except Exception, e: if isinstance(result, _DebugResult): raise result._moduleSetUpFailed = True errorName = 'setUpModule (%s)' % currentModule self._addClassOrModuleLevelException(result, e, errorName) finally: _call_if_exists(result, '_restoreStdout') def _addClassOrModuleLevelException(self, result, exception, errorName): error = _ErrorHolder(errorName) addSkip = getattr(result, 'addSkip', None) if addSkip is not None and isinstance(exception, case.SkipTest): addSkip(error, str(exception)) else: result.addError(error, sys.exc_info()) def _handleModuleTearDown(self, result): previousModule = self._get_previous_module(result) if previousModule is None: return if result._moduleSetUpFailed: return try: module = sys.modules[previousModule] except KeyError: return tearDownModule = getattr(module, 'tearDownModule', None) if tearDownModule is not None: _call_if_exists(result, '_setupStdout') try: tearDownModule() except Exception as e: if isinstance(result, _DebugResult): raise errorName = 'tearDownModule (%s)' % previousModule self._addClassOrModuleLevelException(result, e, errorName) finally: _call_if_exists(result, '_restoreStdout') def _tearDownPreviousClass(self, test, result): previousClass = getattr(result, '_previousTestClass', None) currentClass = test.__class__ if currentClass == previousClass: return if getattr(previousClass, '_classSetupFailed', False): return if getattr(result, '_moduleSetUpFailed', False): return if getattr(previousClass, "__unittest_skip__", False): return tearDownClass = getattr(previousClass, 'tearDownClass', None) if tearDownClass is not None: _call_if_exists(result, '_setupStdout') try: tearDownClass() except Exception, e: if isinstance(result, _DebugResult): raise className = util.strclass(previousClass) errorName = 'tearDownClass (%s)' % className self._addClassOrModuleLevelException(result, e, errorName) finally: _call_if_exists(result, '_restoreStdout') class _ErrorHolder(object): """ Placeholder for a TestCase inside a result. As far as a TestResult is concerned, this looks exactly like a unit test. Used to insert arbitrary errors into a test suite run. """ # Inspired by the ErrorHolder from Twisted: # http://twistedmatrix.com/trac/browser/trunk/twisted/trial/runner.py # attribute used by TestResult._exc_info_to_string failureException = None def __init__(self, description): self.description = description def id(self): return self.description def shortDescription(self): return None def __repr__(self): return "" % (self.description,) def __str__(self): return self.id() def run(self, result): # could call result.addError(...) - but this test-like object # shouldn't be run anyway pass def __call__(self, result): return self.run(result) def countTestCases(self): return 0 def _isnotsuite(test): "A crude way to tell apart testcases and suites with duck-typing" try: iter(test) except TypeError: return True return False class _DebugResult(object): "Used by the TestSuite to hold previous class when running in debug." _previousTestClass = None _moduleSetUpFailed = False shouldStop = False ================================================ FILE: third_party/stdlib/unittest_util.py ================================================ """Various utility functions.""" # from collections import namedtuple, OrderedDict import operator _itemgetter = operator.itemgetter _property = property _tuple = tuple __unittest = True _MAX_LENGTH = 80 def safe_repr(obj, short=False): try: result = repr(obj) except Exception: result = object.__repr__(obj) if not short or len(result) < _MAX_LENGTH: return result return result[:_MAX_LENGTH] + ' [truncated]...' def strclass(cls): return "%s.%s" % (cls.__module__, cls.__name__) def sorted_list_difference(expected, actual): """Finds elements in only one or the other of two, sorted input lists. Returns a two-element tuple of lists. The first list contains those elements in the "expected" list but not in the "actual" list, and the second contains those elements in the "actual" list but not in the "expected" list. Duplicate elements in either input list are ignored. """ i = j = 0 missing = [] unexpected = [] while True: try: e = expected[i] a = actual[j] if e < a: missing.append(e) i += 1 while expected[i] == e: i += 1 elif e > a: unexpected.append(a) j += 1 while actual[j] == a: j += 1 else: i += 1 try: while expected[i] == e: i += 1 finally: j += 1 while actual[j] == a: j += 1 except IndexError: missing.extend(expected[i:]) unexpected.extend(actual[j:]) break return missing, unexpected def unorderable_list_difference(expected, actual, ignore_duplicate=False): """Same behavior as sorted_list_difference but for lists of unorderable items (like dicts). As it does a linear search per item (remove) it has O(n*n) performance. """ missing = [] unexpected = [] while expected: item = expected.pop() try: actual.remove(item) except ValueError: missing.append(item) if ignore_duplicate: for lst in expected, actual: try: while True: lst.remove(item) except ValueError: pass if ignore_duplicate: while actual: item = actual.pop() unexpected.append(item) try: while True: actual.remove(item) except ValueError: pass return missing, unexpected # anything left in actual is unexpected return missing, actual # _Mismatch = namedtuple('Mismatch', 'actual expected value') class _Mismatch(tuple): 'Mismatch(actual, expected, value)' __slots__ = () _fields = ('actual', 'expected', 'value') def __new__(_cls, actual, expected, value): 'Create new instance of Mismatch(actual, expected, value)' return _tuple.__new__(_cls, (actual, expected, value)) # @classmethod def _make(cls, iterable, new=tuple.__new__, len=len): 'Make a new Mismatch object from a sequence or iterable' result = new(cls, iterable) if len(result) != 3: raise TypeError('Expected 3 arguments, got %d' % len(result)) return result _make = classmethod(_make) def __repr__(self): 'Return a nicely formatted representation string' return 'Mismatch(actual=%r, expected=%r, value=%r)' % self def _asdict(self): 'Return a new OrderedDict which maps field names to their values' # return OrderedDict(zip(self._fields, self)) return dict(zip(self._fields, self)) def _replace(_self, **kwds): 'Return a new Mismatch object replacing specified fields with new values' result = _self._make(map(kwds.pop, ('actual', 'expected', 'value'), _self)) if kwds: raise ValueError('Got unexpected field names: %r' % kwds.keys()) return result def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) __dict__ = _property(_asdict) def __getstate__(self): 'Exclude the OrderedDict from pickling' pass actual = _property(_itemgetter(0), doc='Alias for field number 0') expected = _property(_itemgetter(1), doc='Alias for field number 1') value = _property(_itemgetter(2), doc='Alias for field number 2') def _count_diff_all_purpose(actual, expected): 'Returns list of (cnt_act, cnt_exp, elem) triples where the counts differ' # elements need not be hashable s, t = list(actual), list(expected) m, n = len(s), len(t) NULL = object() result = [] for i, elem in enumerate(s): if elem is NULL: continue cnt_s = cnt_t = 0 for j in range(i, m): if s[j] == elem: cnt_s += 1 s[j] = NULL for j, other_elem in enumerate(t): if other_elem == elem: cnt_t += 1 t[j] = NULL if cnt_s != cnt_t: diff = _Mismatch(cnt_s, cnt_t, elem) result.append(diff) for i, elem in enumerate(t): if elem is NULL: continue cnt_t = 0 for j in range(i, n): if t[j] == elem: cnt_t += 1 t[j] = NULL diff = _Mismatch(0, cnt_t, elem) result.append(diff) return result def _ordered_count(iterable): 'Return dict of element counts, in the order they were first seen' c = {} #OrderedDict() for elem in iterable: c[elem] = c.get(elem, 0) + 1 return c def _count_diff_hashable(actual, expected): 'Returns list of (cnt_act, cnt_exp, elem) triples where the counts differ' # elements must be hashable s, t = _ordered_count(actual), _ordered_count(expected) result = [] for elem, cnt_s in s.items(): cnt_t = t.get(elem, 0) if cnt_s != cnt_t: diff = _Mismatch(cnt_s, cnt_t, elem) result.append(diff) for elem, cnt_t in t.items(): if elem not in s: diff = _Mismatch(0, cnt_t, elem) result.append(diff) return result ================================================ FILE: third_party/stdlib/urlparse.py ================================================ """Parse (absolute and relative) URLs. urlparse module is based upon the following RFC specifications. RFC 3986 (STD66): "Uniform Resource Identifiers" by T. Berners-Lee, R. Fielding and L. Masinter, January 2005. RFC 2732 : "Format for Literal IPv6 Addresses in URL's by R.Hinden, B.Carpenter and L.Masinter, December 1999. RFC 2396: "Uniform Resource Identifiers (URI)": Generic Syntax by T. Berners-Lee, R. Fielding, and L. Masinter, August 1998. RFC 2368: "The mailto URL scheme", by P.Hoffman , L Masinter, J. Zwinski, July 1998. RFC 1808: "Relative Uniform Resource Locators", by R. Fielding, UC Irvine, June 1995. RFC 1738: "Uniform Resource Locators (URL)" by T. Berners-Lee, L. Masinter, M. McCahill, December 1994 RFC 3986 is considered the current standard and any future changes to urlparse module should conform with it. The urlparse module is currently not entirely compliant with this RFC due to defacto scenarios for parsing, and for backward compatibility purposes, some parsing quirks from older RFCs are retained. The testcases in test_urlparse.py provides a good indicator of parsing behavior. """ import re import operator _itemgetter = operator.itemgetter _property = property _tuple = tuple __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "parse_qs", "parse_qsl"] # A classification of schemes ('' means apply by default) uses_relative = ['ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', 'prospero', 'rtsp', 'rtspu', '', 'sftp', 'svn', 'svn+ssh'] uses_netloc = ['ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', '', 'svn', 'svn+ssh', 'sftp','nfs','git', 'git+ssh'] uses_params = ['ftp', 'hdl', 'prospero', 'http', 'imap', 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', 'mms', '', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] uses_query = ['http', 'wais', 'imap', 'https', 'shttp', 'mms', 'gopher', 'rtsp', 'rtspu', 'sip', 'sips', ''] uses_fragment = ['ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', 'file', 'prospero', ''] # Characters valid in scheme names scheme_chars = ('abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' '0123456789' '+-.') MAX_CACHE_SIZE = 20 _parse_cache = {} def clear_cache(): """Clear the parse cache.""" _parse_cache.clear() class ResultMixin(object): """Shared methods for the parsed result objects.""" # @property def username(self): netloc = self.netloc if "@" in netloc: userinfo = netloc.rsplit("@", 1)[0] if ":" in userinfo: userinfo = userinfo.split(":", 1)[0] return userinfo return None username = property(username) # @property def password(self): netloc = self.netloc if "@" in netloc: userinfo = netloc.rsplit("@", 1)[0] if ":" in userinfo: return userinfo.split(":", 1)[1] return None password = property(password) # @property def hostname(self): netloc = self.netloc.split('@')[-1] if '[' in netloc and ']' in netloc: return netloc.split(']')[0][1:].lower() elif ':' in netloc: return netloc.split(':')[0].lower() elif netloc == '': return None else: return netloc.lower() hostname = property(hostname) # @property def port(self): netloc = self.netloc.split('@')[-1].split(']')[-1] if ':' in netloc: port = netloc.split(':')[1] if port: port = int(port, 10) # verify legal port if (0 <= port <= 65535): return port return None port = property(port) # from collections import namedtuple class _SplitResult(tuple): 'SplitResult(scheme, netloc, path, query, fragment)' __slots__ = () _fields = ('scheme', 'netloc', 'path', 'query', 'fragment') def __new__(_cls, scheme, netloc, path, query, fragment): 'Create new instance of SplitResult(scheme, netloc, path, query, fragment)' return _tuple.__new__(_cls, (scheme, netloc, path, query, fragment)) # @classmethod def _make(cls, iterable, new=tuple.__new__, len=len): 'Make a new SplitResult object from a sequence or iterable' result = new(cls, iterable) if len(result) != 5: raise TypeError('Expected 5 arguments, got %d' % len(result)) return result _make = classmethod(_make) def __repr__(self): 'Return a nicely formatted representation string' return 'SplitResult(scheme=%r, netloc=%r, path=%r, query=%r, fragment=%r)' % self def _asdict(self): 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) def _replace(_self, **kwds): 'Return a new SplitResult object replacing specified fields with new values' result = _self._make(map(kwds.pop, ('scheme', 'netloc', 'path', 'query', 'fragment'), _self)) if kwds: raise ValueError('Got unexpected field names: %r' % kwds.keys()) return result def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) __dict__ = _property(_asdict) def __getstate__(self): 'Exclude the OrderedDict from pickling' pass scheme = _property(_itemgetter(0), doc='Alias for field number 0') netloc = _property(_itemgetter(1), doc='Alias for field number 1') path = _property(_itemgetter(2), doc='Alias for field number 2') query = _property(_itemgetter(3), doc='Alias for field number 3') fragment = _property(_itemgetter(4), doc='Alias for field number 4') # class SplitResult(namedtuple('SplitResult', 'scheme netloc path query fragment'), ResultMixin): class SplitResult(_SplitResult, ResultMixin): __slots__ = () def geturl(self): return urlunsplit(self) class _ParseResult(tuple): 'ParseResult(scheme, netloc, path, params, query, fragment)' __slots__ = () _fields = ('scheme', 'netloc', 'path', 'params', 'query', 'fragment') def __new__(_cls, scheme, netloc, path, params, query, fragment): 'Create new instance of ParseResult(scheme, netloc, path, params, query, fragment)' return _tuple.__new__(_cls, (scheme, netloc, path, params, query, fragment)) # @classmethod def _make(cls, iterable, new=tuple.__new__, len=len): 'Make a new ParseResult object from a sequence or iterable' result = new(cls, iterable) if len(result) != 6: raise TypeError('Expected 6 arguments, got %d' % len(result)) return result _make = classmethod(_make) def __repr__(self): 'Return a nicely formatted representation string' return 'ParseResult(scheme=%r, netloc=%r, path=%r, params=%r, query=%r, fragment=%r)' % self def _asdict(self): 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) def _replace(_self, **kwds): 'Return a new ParseResult object replacing specified fields with new values' result = _self._make(map(kwds.pop, ('scheme', 'netloc', 'path', 'params', 'query', 'fragment'), _self)) if kwds: raise ValueError('Got unexpected field names: %r' % kwds.keys()) return result def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) __dict__ = _property(_asdict) def __getstate__(self): 'Exclude the OrderedDict from pickling' pass scheme = _property(_itemgetter(0), doc='Alias for field number 0') netloc = _property(_itemgetter(1), doc='Alias for field number 1') path = _property(_itemgetter(2), doc='Alias for field number 2') params = _property(_itemgetter(3), doc='Alias for field number 3') query = _property(_itemgetter(4), doc='Alias for field number 4') fragment = _property(_itemgetter(5), doc='Alias for field number 5') # class ParseResult(namedtuple('ParseResult', 'scheme netloc path params query fragment'), ResultMixin): class ParseResult(_ParseResult, ResultMixin): __slots__ = () def geturl(self): return urlunparse(self) def urlparse(url, scheme='', allow_fragments=True): """Parse a URL into 6 components: :///;?# Return a 6-tuple: (scheme, netloc, path, params, query, fragment). Note that we don't break the components up in smaller bits (e.g. netloc is a single string) and we don't expand % escapes.""" tuple = urlsplit(url, scheme, allow_fragments) scheme, netloc, url, query, fragment = tuple if scheme in uses_params and ';' in url: url, params = _splitparams(url) else: params = '' return ParseResult(scheme, netloc, url, params, query, fragment) def _splitparams(url): if '/' in url: i = url.find(';', url.rfind('/')) if i < 0: return url, '' else: i = url.find(';') return url[:i], url[i+1:] def _splitnetloc(url, start=0): delim = len(url) # position of end of domain part of url, default is end for c in '/?#': # look for delimiters; the order is NOT important wdelim = url.find(c, start) # find first of this delim if wdelim >= 0: # if found delim = min(delim, wdelim) # use earliest delim position return url[start:delim], url[delim:] # return (domain, rest) def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: :///?# Return a 5-tuple: (scheme, netloc, path, query, fragment). Note that we don't break the components up in smaller bits (e.g. netloc is a single string) and we don't expand % escapes.""" allow_fragments = bool(allow_fragments) key = url, scheme, allow_fragments, type(url), type(scheme) cached = _parse_cache.get(key, None) if cached: return cached if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth clear_cache() netloc = query = fragment = '' i = url.find(':') if i > 0: if url[:i] == 'http': # optimize the common case scheme = url[:i].lower() url = url[i+1:] if url[:2] == '//': netloc, url = _splitnetloc(url, 2) if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) v = SplitResult(scheme, netloc, url, query, fragment) _parse_cache[key] = v return v for c in url[:i]: if c not in scheme_chars: break else: # make sure "url" is not actually a port number (in which case # "scheme" is really part of the path) rest = url[i+1:] if not rest or any(c not in '0123456789' for c in rest): # not a port number scheme, url = url[:i].lower(), rest if url[:2] == '//': netloc, url = _splitnetloc(url, 2) if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) v = SplitResult(scheme, netloc, url, query, fragment) _parse_cache[key] = v return v def urlunparse(data): """Put a parsed URL back together again. This may result in a slightly different, but equivalent URL, if the URL that was parsed originally had redundant delimiters, e.g. a ? with an empty query (the draft states that these are equivalent).""" scheme, netloc, url, params, query, fragment = data if params: url = "%s;%s" % (url, params) return urlunsplit((scheme, netloc, url, query, fragment)) def urlunsplit(data): """Combine the elements of a tuple as returned by urlsplit() into a complete URL as a string. The data argument can be any five-item iterable. This may result in a slightly different, but equivalent URL, if the URL that was parsed originally had unnecessary delimiters (for example, a ? with an empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment = data if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): if url and url[:1] != '/': url = '/' + url url = '//' + (netloc or '') + url if scheme: url = scheme + ':' + url if query: url = url + '?' + query if fragment: url = url + '#' + fragment return url def urljoin(base, url, allow_fragments=True): """Join a base URL and a possibly relative URL to form an absolute interpretation of the latter.""" if not base: return url if not url: return base bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ urlparse(base, '', allow_fragments) scheme, netloc, path, params, query, fragment = \ urlparse(url, bscheme, allow_fragments) if scheme != bscheme or scheme not in uses_relative: return url if scheme in uses_netloc: if netloc: return urlunparse((scheme, netloc, path, params, query, fragment)) netloc = bnetloc if path[:1] == '/': return urlunparse((scheme, netloc, path, params, query, fragment)) if not path and not params: path = bpath params = bparams if not query: query = bquery return urlunparse((scheme, netloc, path, params, query, fragment)) segments = bpath.split('/')[:-1] + path.split('/') # XXX The stuff below is bogus in various ways... if segments[-1] == '.': segments[-1] = '' while '.' in segments: segments.remove('.') while 1: i = 1 n = len(segments) - 1 while i < n: if (segments[i] == '..' and segments[i-1] not in ('', '..')): del segments[i-1:i+1] break i = i+1 else: break if segments == ['', '..']: segments[-1] = '' elif len(segments) >= 2 and segments[-1] == '..': segments[-2:] = [''] return urlunparse((scheme, netloc, '/'.join(segments), params, query, fragment)) def urldefrag(url): """Removes any existing fragment from URL. Returns a tuple of the defragmented URL and the fragment. If the URL contained no fragments, the second element is the empty string. """ if '#' in url: s, n, p, a, q, frag = urlparse(url) defrag = urlunparse((s, n, p, a, q, '')) return defrag, frag else: return url, '' try: unicode except NameError: def _is_unicode(x): return 0 else: def _is_unicode(x): return isinstance(x, unicode) # unquote method for parse_qs and parse_qsl # Cannot use directly from urllib as it would create a circular reference # because urllib uses urlparse methods (urljoin). If you update this function, # update it also in urllib. This code duplication does not existin in Python3. _hexdig = '0123456789ABCDEFabcdef' _hextochr = dict((a+b, chr(int(a+b,16))) for a in _hexdig for b in _hexdig) _asciire = re.compile('([\x00-\x7f]+)') def unquote(s): """unquote('abc%20def') -> 'abc def'.""" if _is_unicode(s): if '%' not in s: return s bits = _asciire.split(s) res = [bits[0]] append = res.append for i in range(1, len(bits), 2): append(unquote(str(bits[i])).decode('latin1')) append(bits[i + 1]) return ''.join(res) bits = s.split('%') # fastpath if len(bits) == 1: return s res = [bits[0]] append = res.append for item in bits[1:]: try: append(_hextochr[item[:2]]) append(item[2:]) except KeyError: append('%') append(item) return ''.join(res) def parse_qs(qs, keep_blank_values=0, strict_parsing=0): """Parse a query given as a string argument. Arguments: qs: percent-encoded query string to be parsed keep_blank_values: flag indicating whether blank values in percent-encoded queries should be treated as blank strings. A true value indicates that blanks should be retained as blank strings. The default false value indicates that blank values are to be ignored and treated as if they were not included. strict_parsing: flag indicating what to do with parsing errors. If false (the default), errors are silently ignored. If true, errors raise a ValueError exception. """ dict = {} for name, value in parse_qsl(qs, keep_blank_values, strict_parsing): if name in dict: dict[name].append(value) else: dict[name] = [value] return dict def parse_qsl(qs, keep_blank_values=0, strict_parsing=0): """Parse a query given as a string argument. Arguments: qs: percent-encoded query string to be parsed keep_blank_values: flag indicating whether blank values in percent-encoded queries should be treated as blank strings. A true value indicates that blanks should be retained as blank strings. The default false value indicates that blank values are to be ignored and treated as if they were not included. strict_parsing: flag indicating what to do with parsing errors. If false (the default), errors are silently ignored. If true, errors raise a ValueError exception. Returns a list, as G-d intended. """ pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] r = [] for name_value in pairs: if not name_value and not strict_parsing: continue nv = name_value.split('=', 1) if len(nv) != 2: if strict_parsing: raise ValueError, "bad query field: %r" % (name_value,) # Handle case of a control-name with no equal sign if keep_blank_values: nv.append('') else: continue if len(nv[1]) or keep_blank_values: name = unquote(nv[0].replace('+', ' ')) value = unquote(nv[1].replace('+', ' ')) r.append((name, value)) return r ================================================ FILE: third_party/stdlib/uu.py ================================================ #! /usr/bin/env python # Copyright 1994 by Lance Ellinghouse # Cathedral City, California Republic, United States of America. # All Rights Reserved # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, # provided that the above copyright notice appear in all copies and that # both that copyright notice and this permission notice appear in # supporting documentation, and that the name of Lance Ellinghouse # not be used in advertising or publicity pertaining to distribution # of the software without specific, written prior permission. # LANCE ELLINGHOUSE DISCLAIMS ALL WARRANTIES WITH REGARD TO # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND # FITNESS, IN NO EVENT SHALL LANCE ELLINGHOUSE CENTRUM BE LIABLE # FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # # Modified by Jack Jansen, CWI, July 1995: # - Use binascii module to do the actual line-by-line conversion # between ascii and binary. This results in a 1000-fold speedup. The C # version is still 5 times faster, though. # - Arguments more compliant with python standard """Implementation of the UUencode and UUdecode functions. encode(in_file, out_file [,name, mode]) decode(in_file [, out_file, mode]) """ import binascii import os import sys __all__ = ["Error", "encode", "decode"] class Error(Exception): pass def encode(in_file, out_file, name=None, mode=None): """Uuencode file""" # # If in_file is a pathname open it and change defaults # opened_files = [] try: if in_file == '-': in_file = sys.stdin elif isinstance(in_file, basestring): if name is None: name = os.path.basename(in_file) if mode is None: try: mode = os.stat(in_file).st_mode except AttributeError: pass in_file = open(in_file, 'rb') opened_files.append(in_file) # # Open out_file if it is a pathname # if out_file == '-': out_file = sys.stdout elif isinstance(out_file, basestring): out_file = open(out_file, 'wb') opened_files.append(out_file) # # Set defaults for name and mode # if name is None: name = '-' if mode is None: mode = 0666 # # Write the data # out_file.write('begin %o %s\n' % ((mode&0777),name)) data = in_file.read(45) while len(data) > 0: out_file.write(binascii.b2a_uu(data)) data = in_file.read(45) out_file.write(' \nend\n') finally: for f in opened_files: f.close() def decode(in_file, out_file=None, mode=None, quiet=0): """Decode uuencoded file""" # # Open the input file, if needed. # opened_files = [] if in_file == '-': in_file = sys.stdin elif isinstance(in_file, basestring): in_file = open(in_file) opened_files.append(in_file) try: # # Read until a begin is encountered or we've exhausted the file # while True: hdr = in_file.readline() if not hdr: raise Error('No valid begin line found in input file') if not hdr.startswith('begin'): continue hdrfields = hdr.split(' ', 2) if len(hdrfields) == 3 and hdrfields[0] == 'begin': try: int(hdrfields[1], 8) break except ValueError: pass if out_file is None: out_file = hdrfields[2].rstrip() if os.path.exists(out_file): raise Error('Cannot overwrite existing file: %s' % out_file) if mode is None: mode = int(hdrfields[1], 8) # # Open the output file # if out_file == '-': out_file = sys.stdout elif isinstance(out_file, basestring): fp = open(out_file, 'wb') try: os.path.chmod(out_file, mode) except AttributeError: pass out_file = fp opened_files.append(out_file) # # Main decoding loop # s = in_file.readline() while s and s.strip() != 'end': try: data = binascii.a2b_uu(s) except binascii.Error, v: # Workaround for broken uuencoders by /Fredrik Lundh nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3 data = binascii.a2b_uu(s[:nbytes]) if not quiet: sys.stderr.write("Warning: %s\n" % v) out_file.write(data) s = in_file.readline() if not s: raise Error('Truncated input file') finally: for f in opened_files: f.close() def test(): """uuencode/uudecode main program""" import optparse parser = optparse.OptionParser(usage='usage: %prog [-d] [-t] [input [output]]') parser.add_option('-d', '--decode', dest='decode', help='Decode (instead of encode)?', default=False, action='store_true') parser.add_option('-t', '--text', dest='text', help='data is text, encoded format unix-compatible text?', default=False, action='store_true') (options, args) = parser.parse_args() if len(args) > 2: parser.error('incorrect number of arguments') sys.exit(1) input = sys.stdin output = sys.stdout if len(args) > 0: input = args[0] if len(args) > 1: output = args[1] if options.decode: if options.text: if isinstance(output, basestring): output = open(output, 'w') else: print sys.argv[0], ': cannot do -t to stdout' sys.exit(1) decode(input, output) else: if options.text: if isinstance(input, basestring): input = open(input, 'r') else: print sys.argv[0], ': cannot do -t from stdin' sys.exit(1) encode(input, output) if __name__ == '__main__': test() ================================================ FILE: third_party/stdlib/warnings.py ================================================ """Python part of the warnings subsystem.""" # Note: function level imports should *not* be used # in this module as it may cause import lock deadlock. # See bug 683658. import linecache import re import sys import types __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings", "warnpy3k"] def warnpy3k(message, category=None, stacklevel=1): """Issue a deprecation warning for Python 3.x related changes. Warnings are omitted unless Python is started with the -3 option. """ if sys.py3kwarning: if category is None: category = DeprecationWarning warn(message, category, stacklevel+1) def _show_warning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" if file is None: file = sys.stderr if file is None: # sys.stderr is None - warnings get lost return try: file.write(formatwarning(message, category, filename, lineno, line)) except (IOError, UnicodeError): pass # the file (probably stderr) is invalid - this warning gets lost. # Keep a working version around in case the deprecation of the old API is # triggered. showwarning = _show_warning def formatwarning(message, category, filename, lineno, line=None): """Function to format a warning the standard way.""" try: unicodetype = unicode except NameError: unicodetype = () try: message = str(message) except UnicodeEncodeError: pass s = "%s: %s: %s\n" % (lineno, category.__name__, message) line = linecache.getline(filename, lineno) if line is None else line if line: line = line.strip() if isinstance(s, unicodetype) and isinstance(line, str): line = unicode(line, 'latin1') s += " %s\n" % line if isinstance(s, unicodetype) and isinstance(filename, str): enc = sys.getfilesystemencoding() if enc: try: filename = unicode(filename, enc) except UnicodeDecodeError: pass s = "%s:%s" % (filename, s) return s def filterwarnings(action, message="", category=Warning, module="", lineno=0, append=0): """Insert an entry into the list of warnings filters (at the front). 'action' -- one of "error", "ignore", "always", "default", "module", or "once" 'message' -- a regex that the warning message must match 'category' -- a class that the warning must be a subclass of 'module' -- a regex that the module name must match 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ assert action in ("error", "ignore", "always", "default", "module", "once"), "invalid action: %r" % (action,) assert isinstance(message, basestring), "message must be a string" assert isinstance(category, type), "category must be a class" assert issubclass(category, Warning), "category must be a Warning subclass" assert isinstance(module, basestring), "module must be a string" assert isinstance(lineno, int) and lineno >= 0, \ "lineno must be an int >= 0" item = (action, re.compile(message, re.I), category, re.compile(module), lineno) if append: filters.append(item) else: filters.insert(0, item) def simplefilter(action, category=Warning, lineno=0, append=0): """Insert a simple entry into the list of warnings filters (at the front). A simple filter matches all modules and messages. 'action' -- one of "error", "ignore", "always", "default", "module", or "once" 'category' -- a class that the warning must be a subclass of 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ assert action in ("error", "ignore", "always", "default", "module", "once"), "invalid action: %r" % (action,) assert isinstance(lineno, int) and lineno >= 0, \ "lineno must be an int >= 0" item = (action, None, category, None, lineno) if append: filters.append(item) else: filters.insert(0, item) def resetwarnings(): """Clear the list of warning filters, so that no filters are active.""" filters[:] = [] class _OptionError(Exception): """Exception used by option processing helpers.""" pass # Helper to process -W options passed via sys.warnoptions def _processoptions(args): for arg in args: try: _setoption(arg) except _OptionError, msg: print >>sys.stderr, "Invalid -W option ignored:", msg # Helper for _processoptions() def _setoption(arg): parts = arg.split(':') if len(parts) > 5: raise _OptionError("too many fields (max 5): %r" % (arg,)) while len(parts) < 5: parts.append('') action, message, category, module, lineno = [s.strip() for s in parts] action = _getaction(action) message = re.escape(message) category = _getcategory(category) module = re.escape(module) if module: module = module + '$' if lineno: try: lineno = int(lineno) if lineno < 0: raise ValueError except (ValueError, OverflowError): raise _OptionError("invalid lineno %r" % (lineno,)) else: lineno = 0 filterwarnings(action, message, category, module, lineno) # Helper for _setoption() def _getaction(action): if not action: return "default" if action == "all": return "always" # Alias for a in ('default', 'always', 'ignore', 'module', 'once', 'error'): if a.startswith(action): return a raise _OptionError("invalid action: %r" % (action,)) # Helper for _setoption() def _getcategory(category): if not category: return Warning if re.match("^[a-zA-Z0-9_]+$", category): try: cat = eval(category) except NameError: raise _OptionError("unknown warning category: %r" % (category,)) else: i = category.rfind(".") module = category[:i] klass = category[i+1:] try: m = __import__(module, None, None, [klass]) except ImportError: raise _OptionError("invalid module name: %r" % (module,)) try: cat = getattr(m, klass) except AttributeError: raise _OptionError("unknown warning category: %r" % (category,)) if not issubclass(cat, Warning): raise _OptionError("invalid warning category: %r" % (category,)) return cat # Code typically replaced by _warnings def warn(message, category=None, stacklevel=1): """Issue a warning, or maybe ignore it or raise an exception.""" # Check if message is already a Warning object if isinstance(message, Warning): category = message.__class__ # Check category argument if category is None: category = UserWarning assert issubclass(category, Warning) # Get context information try: caller = sys._getframe(stacklevel) except ValueError: globals = sys.__dict__ lineno = 1 else: globals = caller.f_globals lineno = caller.f_lineno if '__name__' in globals: module = globals['__name__'] else: module = "" filename = globals.get('__file__') if filename: fnl = filename.lower() if fnl.endswith((".pyc", ".pyo")): filename = filename[:-1] else: if module == "__main__": try: filename = sys.argv[0] except AttributeError: # embedded interpreters don't have sys.argv, see bug #839151 filename = '__main__' if not filename: filename = module registry = globals.setdefault("__warningregistry__", {}) warn_explicit(message, category, filename, lineno, module, registry, globals) def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None): lineno = int(lineno) if module is None: module = filename or "" if module[-3:].lower() == ".py": module = module[:-3] # XXX What about leading pathname? if registry is None: registry = {} if isinstance(message, Warning): text = str(message) category = message.__class__ else: text = message message = category(message) key = (text, category, lineno) # Quick test for common case if registry.get(key): return # Search the filters for item in filters: action, msg, cat, mod, ln = item if ((msg is None or msg.match(text)) and issubclass(category, cat) and (mod is None or mod.match(module)) and (ln == 0 or lineno == ln)): break else: action = defaultaction # Early exit actions if action == "ignore": registry[key] = 1 return # Prime the linecache for formatting, in case the # "file" is actually in a zipfile or something. linecache.getlines(filename, module_globals) if action == "error": raise message # Other actions if action == "once": registry[key] = 1 oncekey = (text, category) if onceregistry.get(oncekey): return onceregistry[oncekey] = 1 elif action == "always": pass elif action == "module": registry[key] = 1 altkey = (text, category, 0) if registry.get(altkey): return registry[altkey] = 1 elif action == "default": registry[key] = 1 else: # Unrecognized actions are errors raise RuntimeError( "Unrecognized action (%r) in warnings.filters:\n %s" % (action, item)) # Print message and context showwarning(message, category, filename, lineno) class WarningMessage(object): """Holds the result of a single showwarning() call.""" _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", "line") def __init__(self, message, category, filename, lineno, file=None, line=None): local_values = locals() for attr in self._WARNING_DETAILS: setattr(self, attr, local_values[attr]) self._category_name = category.__name__ if category else None def __str__(self): return ("{message : %r, category : %r, filename : %r, lineno : %s, " "line : %r}" % (self.message, self._category_name, self.filename, self.lineno, self.line)) class catch_warnings(object): """A context manager that copies and restores the warnings filter upon exiting the context. The 'record' argument specifies whether warnings should be captured by a custom implementation of warnings.showwarning() and be appended to a list returned by the context manager. Otherwise None is returned by the context manager. The objects appended to the list are arguments whose attributes mirror the arguments to showwarning(). The 'module' argument is to specify an alternative module to the module named 'warnings' and imported under that name. This argument is only useful when testing the warnings module itself. """ def __init__(self, record=False, module=None): """Specify whether to record warnings and if an alternative module should be used other than sys.modules['warnings']. For compatibility with Python 3.0, please consider all arguments to be keyword-only. """ self._record = record self._module = sys.modules['warnings'] if module is None else module self._entered = False def __repr__(self): args = [] if self._record: args.append("record=True") if self._module is not sys.modules['warnings']: args.append("module=%r" % self._module) name = type(self).__name__ return "%s(%s)" % (name, ", ".join(args)) def __enter__(self): if self._entered: raise RuntimeError("Cannot enter %r twice" % self) self._entered = True self._filters = self._module.filters self._module.filters = self._filters[:] self._showwarning = self._module.showwarning if self._record: log = [] def showwarning(*args, **kwargs): log.append(WarningMessage(*args, **kwargs)) self._module.showwarning = showwarning return log else: return None def __exit__(self, *exc_info): if not self._entered: raise RuntimeError("Cannot exit %r without entering first" % self) self._module.filters = self._filters self._module.showwarning = self._showwarning # filters contains a sequence of filter 5-tuples # The components of the 5-tuple are: # - an action: error, ignore, always, default, module, or once # - a compiled regex that must match the warning message # - a class representing the warning category # - a compiled regex that must match the module that is being warned # - a line number for the line being warning, or 0 to mean any line # If either if the compiled regexs are None, match anything. _warnings_defaults = False # try: # from _warnings import (filters, default_action, once_registry, # warn, warn_explicit) # defaultaction = default_action # onceregistry = once_registry # _warnings_defaults = True # except ImportError: filters = [] defaultaction = "default" onceregistry = {} # Module initialization _processoptions(sys.warnoptions) if not _warnings_defaults: silence = [ImportWarning, PendingDeprecationWarning] # Don't silence DeprecationWarning if -3 or -Q was used. if not sys.py3kwarning and not sys.flags.division_warning: silence.append(DeprecationWarning) for cls in silence: simplefilter("ignore", category=cls) bytes_warning = sys.flags.bytes_warning if bytes_warning > 1: bytes_action = "error" elif bytes_warning: bytes_action = "default" else: bytes_action = "ignore" simplefilter(bytes_action, category=BytesWarning, append=1) del _warnings_defaults ================================================ FILE: third_party/stdlib/weakref.py ================================================ """Weak reference support for Python. This module is an implementation of PEP 205: http://www.python.org/dev/peps/pep-0205/ """ # Naming convention: Variables named "wr" are weak reference objects; # they are called this instead of "ref" to avoid name collisions with # the module-global ref() function imported from _weakref. import UserDict #from _weakref import ( # getweakrefcount, # getweakrefs, # ref, # proxy, # CallableProxyType, # ProxyType, # ReferenceType) from '__go__/grumpy' import WeakRefType as ReferenceType ref = ReferenceType import _weakrefset WeakSet = _weakrefset.WeakSet _IterationGuard = _weakrefset._IterationGuard import exceptions ReferenceError = exceptions.ReferenceError #ProxyTypes = (ProxyType, CallableProxyType) #__all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs", # "WeakKeyDictionary", "ReferenceError", "ReferenceType", "ProxyType", # "CallableProxyType", "ProxyTypes", "WeakValueDictionary", 'WeakSet'] class WeakValueDictionary(UserDict.UserDict): """Mapping class that references values weakly. Entries in the dictionary will be discarded when no strong reference to the value exists anymore """ # We inherit the constructor without worrying about the input # dictionary; since it uses our .update() method, we get the right # checks (if the other dictionary is a WeakValueDictionary, # objects are unwrapped on the way out, and we always wrap on the # way in). def __init__(*args, **kw): if not args: raise TypeError("descriptor '__init__' of 'WeakValueDictionary' " "object needs an argument") self = args[0] args = args[1:] if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) def remove(wr, selfref=ref(self)): self = selfref() if self is not None: if self._iterating: self._pending_removals.append(wr.key) else: del self.data[wr.key] self._remove = remove # A list of keys to be removed self._pending_removals = [] self._iterating = set() UserDict.UserDict.__init__(self, *args, **kw) def _commit_removals(self): l = self._pending_removals d = self.data # We shouldn't encounter any KeyError, because this method should # always be called *before* mutating the dict. while l: del d[l.pop()] def __getitem__(self, key): o = self.data[key]() if o is None: raise KeyError, key else: return o def __delitem__(self, key): if self._pending_removals: self._commit_removals() del self.data[key] def __contains__(self, key): try: o = self.data[key]() except KeyError: return False return o is not None def has_key(self, key): try: o = self.data[key]() except KeyError: return False return o is not None def __repr__(self): return "" % id(self) def __setitem__(self, key, value): if self._pending_removals: self._commit_removals() self.data[key] = KeyedRef(value, self._remove, key) def clear(self): if self._pending_removals: self._commit_removals() self.data.clear() def copy(self): new = WeakValueDictionary() for key, wr in self.data.items(): o = wr() if o is not None: new[key] = o return new __copy__ = copy def __deepcopy__(self, memo): import copy new = self.__class__() for key, wr in self.data.items(): o = wr() if o is not None: new[copy.deepcopy(key, memo)] = o return new def get(self, key, default=None): try: wr = self.data[key] except KeyError: return default else: o = wr() if o is None: # This should only happen return default else: return o def items(self): L = [] for key, wr in self.data.items(): o = wr() if o is not None: L.append((key, o)) return L def iteritems(self): with _IterationGuard(self): for wr in self.data.itervalues(): value = wr() if value is not None: yield wr.key, value def iterkeys(self): with _IterationGuard(self): for k in self.data.iterkeys(): yield k __iter__ = iterkeys def itervaluerefs(self): """Return an iterator that yields the weak references to the values. The references are not guaranteed to be 'live' at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the values around longer than needed. """ with _IterationGuard(self): for wr in self.data.itervalues(): yield wr def itervalues(self): with _IterationGuard(self): for wr in self.data.itervalues(): obj = wr() if obj is not None: yield obj def popitem(self): if self._pending_removals: self._commit_removals() while 1: key, wr = self.data.popitem() o = wr() if o is not None: return key, o def pop(self, key, *args): if self._pending_removals: self._commit_removals() try: o = self.data.pop(key)() except KeyError: if args: return args[0] raise if o is None: raise KeyError, key else: return o def setdefault(self, key, default=None): try: wr = self.data[key] except KeyError: if self._pending_removals: self._commit_removals() self.data[key] = KeyedRef(default, self._remove, key) return default else: return wr() def update(*args, **kwargs): if not args: raise TypeError("descriptor 'update' of 'WeakValueDictionary' " "object needs an argument") self = args[0] args = args[1:] if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) dict = args[0] if args else None if self._pending_removals: self._commit_removals() d = self.data if dict is not None: if not hasattr(dict, "items"): dict = type({})(dict) for key, o in dict.items(): d[key] = KeyedRef(o, self._remove, key) if len(kwargs): self.update(kwargs) def valuerefs(self): """Return a list of weak references to the values. The references are not guaranteed to be 'live' at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the values around longer than needed. """ return self.data.values() def values(self): L = [] for wr in self.data.values(): o = wr() if o is not None: L.append(o) return L class KeyedRef(ref): """Specialized reference that includes a key corresponding to the value. This is used in the WeakValueDictionary to avoid having to create a function object for each key stored in the mapping. A shared callback object can use the 'key' attribute of a KeyedRef instead of getting a reference to the key from an enclosing scope. """ __slots__ = "key", def __new__(type, ob, callback, key): self = ref.__new__(type, ob, callback) self.key = key return self def __init__(self, ob, callback, key): super(KeyedRef, self).__init__(ob, callback) class WeakKeyDictionary(UserDict.UserDict): """ Mapping class that references keys weakly. Entries in the dictionary will be discarded when there is no longer a strong reference to the key. This can be used to associate additional data with an object owned by other parts of an application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. """ def __init__(self, dict=None): self.data = {} def remove(k, selfref=ref(self)): self = selfref() if self is not None: if self._iterating: self._pending_removals.append(k) else: del self.data[k] self._remove = remove # A list of dead weakrefs (keys to be removed) self._pending_removals = [] self._iterating = set() if dict is not None: self.update(dict) def _commit_removals(self): # NOTE: We don't need to call this method before mutating the dict, # because a dead weakref never compares equal to a live weakref, # even if they happened to refer to equal objects. # However, it means keys may already have been removed. l = self._pending_removals d = self.data while l: try: del d[l.pop()] except KeyError: pass def __delitem__(self, key): del self.data[ref(key)] def __getitem__(self, key): return self.data[ref(key)] def __repr__(self): return "" % id(self) def __setitem__(self, key, value): self.data[ref(key, self._remove)] = value def copy(self): new = WeakKeyDictionary() for key, value in self.data.items(): o = key() if o is not None: new[o] = value return new __copy__ = copy def __deepcopy__(self, memo): import copy new = self.__class__() for key, value in self.data.items(): o = key() if o is not None: new[o] = copy.deepcopy(value, memo) return new def get(self, key, default=None): return self.data.get(ref(key),default) def has_key(self, key): try: wr = ref(key) except TypeError: return 0 return wr in self.data def __contains__(self, key): try: wr = ref(key) except TypeError: return 0 return wr in self.data def items(self): L = [] for key, value in self.data.items(): o = key() if o is not None: L.append((o, value)) return L def iteritems(self): with _IterationGuard(self): for wr, value in self.data.iteritems(): key = wr() if key is not None: yield key, value def iterkeyrefs(self): """Return an iterator that yields the weak references to the keys. The references are not guaranteed to be 'live' at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the keys around longer than needed. """ with _IterationGuard(self): for wr in self.data.iterkeys(): yield wr def iterkeys(self): with _IterationGuard(self): for wr in self.data.iterkeys(): obj = wr() if obj is not None: yield obj __iter__ = iterkeys def itervalues(self): with _IterationGuard(self): for value in self.data.itervalues(): yield value def keyrefs(self): """Return a list of weak references to the keys. The references are not guaranteed to be 'live' at the time they are used, so the result of calling the references needs to be checked before being used. This can be used to avoid creating references that will cause the garbage collector to keep the keys around longer than needed. """ return self.data.keys() def keys(self): L = [] for wr in self.data.keys(): o = wr() if o is not None: L.append(o) return L def popitem(self): while 1: key, value = self.data.popitem() o = key() if o is not None: return o, value def pop(self, key, *args): return self.data.pop(ref(key), *args) def setdefault(self, key, default=None): return self.data.setdefault(ref(key, self._remove),default) def update(self, dict=None, **kwargs): d = self.data if dict is not None: if not hasattr(dict, "items"): dict = type({})(dict) for key, value in dict.items(): d[ref(key, self._remove)] = value if len(kwargs): self.update(kwargs) ================================================ FILE: tools/benchcmp ================================================ #!/usr/bin/env python # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Runs two benchmark programs and compares their results.""" import argparse import subprocess import sys parser = argparse.ArgumentParser() parser.add_argument('prog1') parser.add_argument('prog2') parser.add_argument('--runs', default=1, type=int, help='number of times to run each program') def main(args): results1 = _RunBenchmark(args.prog1) benchmarks = set(results1.keys()) results2 = {} for _ in xrange(args.runs - 1): _MergeResults(results1, _RunBenchmark(args.prog1), benchmarks) _MergeResults(results2, _RunBenchmark(args.prog2), benchmarks) _MergeResults(results2, _RunBenchmark(args.prog2), benchmarks) for b in sorted(benchmarks): print b, '{:+.1%}'.format(results2[b] / results1[b] - 1) def _MergeResults(merged, results, benchmarks): benchmarks = set(benchmarks) for k, v in results.iteritems(): if k not in benchmarks: _Die('unmatched benchmark: {}', k) merged[k] = max(merged.get(k, 0), v) benchmarks.remove(k) if benchmarks: _Die('missing benchmark(s): {}', ', '.join(benchmarks)) def _RunBenchmark(prog): """Executes prog and returns a dict mapping benchmark name -> result.""" try: p = subprocess.Popen([prog], shell=True, stdout=subprocess.PIPE) except OSError as e: _Die(e) out, _ = p.communicate() if p.returncode: _Die('{} exited with status: {}', prog, p.returncode) results = {} for line in out.splitlines(): line = line.strip() if not line: continue parts = line.split() if len(parts) != 3: _Die('invalid benchmark output: {}', line) name, status, result = parts if status != 'PASSED': _Die('benchmark failed: {}', line) try: result = float(result) except ValueError: _Die('invalid benchmark result: {}', line) results[name] = result return results def _Die(msg, *args): if args: msg = msg.format(*args) print >> sys.stderr, msg sys.exit(1) if __name__ == '__main__': main(parser.parse_args()) ================================================ FILE: tools/coverparse ================================================ #!/usr/bin/env python # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Parse a Go coverage file and prints a message for lines missing coverage.""" import collections import re import sys cover_re = re.compile(r'([^:]+):(\d+)\.\d+,(\d+).\d+ \d+ (\d+)$') def _ParseCover(f): """Return a dict of sets with uncovered line numbers from a Go cover file.""" uncovered = collections.defaultdict(set) for line in f: match = cover_re.match(line.rstrip()) if not match: raise RuntimeError('invalid coverage line: {!r}'.format(line)) filename, line_start, line_end, count = match.groups() if not int(count): for i in xrange(int(line_start), int(line_end) + 1): uncovered[filename].add(i) return uncovered def main(): with open(sys.argv[1]) as f: f.readline() uncovered = _ParseCover(f) for filename in sorted(uncovered.keys()): for lineno in sorted(uncovered[filename]): print '{}:{}'.format(filename, lineno) if __name__ == '__main__': main() ================================================ FILE: tools/diffrange ================================================ #!/usr/bin/env python # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Convert a unified diff into a list of modified files and line numbers.""" import sys class _LineBuffer(object): """Iterator over lines in a file supporting one-step rewind.""" def __init__(self, f): self._f = f self._prev = None self._next = None def __iter__(self): return self def next(self): if self._next is not None: cur = self._next else: cur = self._f.readline() if not cur: raise StopIteration self._next = None self._prev = cur return cur def Rewind(self): assert self._prev is not None self._next = self._prev self._prev = None def _ReadHunks(buf): for line in buf: if not line.startswith('@@'): break base = int(line.split()[2].split(',')[0]) for offset in _ReadHunkBody(buf): yield base + offset def _ReadHunkBody(buf): n = 0 for line in buf: prefix = line[0] if prefix == ' ': n += 1 elif prefix == '+': yield n n += 1 elif prefix != '-' or line.startswith('---'): buf.Rewind() break def main(): buf = _LineBuffer(sys.stdin) for line in buf: if line.startswith('+++'): filename = line.split()[1] for n in _ReadHunks(buf): print '{}:{}'.format(filename, n) if __name__ == '__main__': main() ================================================ FILE: tools/genmake ================================================ #!/usr/bin/env python # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Generate a Makefile for Python targets in a GOPATH directory.""" import argparse import os import subprocess import sys parser = argparse.ArgumentParser() parser.add_argument('dir', help='GOPATH dir to scan for Python modules') parser.add_argument('-all_target', default='all', help='make target that will build all modules') def _PrintRule(target, prereqs, rules): print '{}: {}'.format(target, ' '.join(prereqs)) if rules: print '\t@mkdir -p $(@D)' for rule in rules: print '\t@{}'.format(rule) print def main(args): try: proc = subprocess.Popen('go env GOOS GOARCH', shell=True, stdout=subprocess.PIPE) except OSError as e: print >> sys.stderr, str(e) return 1 out, _ = proc.communicate() if proc.returncode: print >> sys.stderr, 'go exited with status: {}'.format(proc.returncode) return 1 goos, goarch = out.split() if args.all_target: print '{}:\n'.format(args.all_target) gopath = os.path.normpath(args.dir) pkg_dir = os.path.join(gopath, 'pkg', '{}_{}'.format(goos, goarch)) pydir = os.path.join(gopath, 'src', '__python__') for dirpath, _, filenames in os.walk(pydir): for filename in filenames: if not filename.endswith('.py'): continue basename = os.path.relpath(dirpath, pydir) if filename != '__init__.py': basename = os.path.normpath( os.path.join(basename, filename[:-3])) modname = basename.replace(os.sep, '.') ar_name = os.path.join(pkg_dir, '__python__', basename + '.a') go_file = os.path.join(pydir, basename, 'module.go') _PrintRule(go_file, [os.path.join(dirpath, filename)], ['grumpc -modname={} $< > $@'.format(modname)]) recipe = (r"""pydeps -modname=%s $< | awk '{gsub(/\./, "/", $$0); """ r"""print "%s: %s/__python__/" $$0 ".a"}' > $@""") dep_file = os.path.join(pydir, basename, 'module.d') _PrintRule(dep_file, [os.path.join(dirpath, filename)], [recipe % (modname, ar_name, pkg_dir)]) go_package = '__python__/' + basename.replace(os.sep, '/') recipe = 'go tool compile -o $@ -p {} -complete -I {} -pack $<' _PrintRule(ar_name, [go_file], [recipe.format(go_package, pkg_dir)]) if args.all_target: _PrintRule(args.all_target, [ar_name], []) print '-include {}\n'.format(dep_file) if __name__ == '__main__': sys.exit(main(parser.parse_args())) ================================================ FILE: tools/grumpc ================================================ #!/usr/bin/env python # coding=utf-8 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A Python -> Go transcompiler.""" from __future__ import unicode_literals import argparse import os import sys import textwrap from grumpy.compiler import block from grumpy.compiler import imputil from grumpy.compiler import stmt from grumpy.compiler import util from grumpy import pythonparser parser = argparse.ArgumentParser() parser.add_argument('script', help='Python source filename') parser.add_argument('-modname', default='__main__', help='Python module name') def main(args): for arg in ('script', 'modname'): if not getattr(args, arg, None): print >> sys.stderr, '{} arg must not be empty'.format(arg) return 1 gopath = os.getenv('GOPATH', None) if not gopath: print >> sys.stderr, 'GOPATH not set' return 1 with open(args.script) as py_file: py_contents = py_file.read() try: mod = pythonparser.parse(py_contents) except SyntaxError as e: print >> sys.stderr, '{}: line {}: invalid syntax: {}'.format( e.filename, e.lineno, e.text) return 2 # Do a pass for compiler directives from `from __future__ import *` statements try: future_node, future_features = imputil.parse_future_features(mod) except util.CompileError as e: print >> sys.stderr, str(e) return 2 importer = imputil.Importer(gopath, args.modname, args.script, future_features.absolute_import) full_package_name = args.modname.replace('.', '/') mod_block = block.ModuleBlock(importer, full_package_name, args.script, py_contents, future_features) visitor = stmt.StatementVisitor(mod_block, future_node) # Indent so that the module body is aligned with the goto labels. with visitor.writer.indent_block(): try: visitor.visit(mod) except util.ParseError as e: print >> sys.stderr, str(e) return 2 writer = util.Writer(sys.stdout) tmpl = textwrap.dedent("""\ package $package import πg "grumpy" var Code *πg.Code func init() { \tCode = πg.NewCode("", $script, nil, 0, func(πF *πg.Frame, _ []*πg.Object) (*πg.Object, *πg.BaseException) { \t\tvar πR *πg.Object; _ = πR \t\tvar πE *πg.BaseException; _ = πE""") writer.write_tmpl(tmpl, package=args.modname.split('.')[-1], script=util.go_str(args.script)) with writer.indent_block(2): for s in sorted(mod_block.strings): writer.write('ß{} := πg.InternStr({})'.format(s, util.go_str(s))) writer.write_temp_decls(mod_block) writer.write_block(mod_block, visitor.writer.getvalue()) writer.write_tmpl(textwrap.dedent("""\ \t\treturn nil, πE \t}) \tπg.RegisterModule($modname, Code) }"""), modname=util.go_str(args.modname)) return 0 if __name__ == '__main__': sys.exit(main(parser.parse_args())) ================================================ FILE: tools/grumprun ================================================ #!/usr/bin/env python # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """grumprun compiles and runs a snippet of Python using Grumpy. Usage: $ grumprun -m # Run the named module. $ echo 'print "hola!"' | grumprun # Execute Python code from stdin. """ import argparse import os import random import shutil import string import subprocess import sys import tempfile from grumpy.compiler import imputil parser = argparse.ArgumentParser() parser.add_argument('-m', '--modname', help='Run the named module') module_tmpl = string.Template("""\ package main import ( \t"os" \t"grumpy" \tmod "$package" $imports ) func main() { \tgrumpy.ImportModule(grumpy.NewRootFrame(), "traceback") \tos.Exit(grumpy.RunMain(mod.Code)) } """) def main(args): gopath = os.getenv('GOPATH', None) if not gopath: print >> sys.stderr, 'GOPATH not set' return 1 modname = args.modname workdir = tempfile.mkdtemp() try: if modname: # Find the script associated with the given module. for d in gopath.split(os.pathsep): script = imputil.find_script( os.path.join(d, 'src', '__python__'), modname) if script: break else: print >> sys.stderr, "can't find module", modname return 1 else: # Generate a dummy python script on the GOPATH. modname = ''.join(random.choice(string.ascii_letters) for _ in range(16)) py_dir = os.path.join(workdir, 'src', '__python__') mod_dir = os.path.join(py_dir, modname) os.makedirs(mod_dir) script = os.path.join(py_dir, 'module.py') with open(script, 'w') as f: f.write(sys.stdin.read()) gopath = gopath + os.pathsep + workdir os.putenv('GOPATH', gopath) # Compile the dummy script to Go using grumpc. fd = os.open(os.path.join(mod_dir, 'module.go'), os.O_WRONLY | os.O_CREAT) try: p = subprocess.Popen('grumpc ' + script, stdout=fd, shell=True) if p.wait(): return 1 finally: os.close(fd) names = imputil.calculate_transitive_deps(modname, script, gopath) # Make sure traceback is available in all Python binaries. names.add('traceback') go_main = os.path.join(workdir, 'main.go') package = _package_name(modname) imports = ''.join('\t_ "' + _package_name(name) + '"\n' for name in names) with open(go_main, 'w') as f: f.write(module_tmpl.substitute(package=package, imports=imports)) return subprocess.Popen('go run ' + go_main, shell=True).wait() finally: shutil.rmtree(workdir) def _package_name(modname): if modname.startswith('__go__/'): return '__python__/' + modname return '__python__/' + modname.replace('.', '/') if __name__ == '__main__': sys.exit(main(parser.parse_args())) ================================================ FILE: tools/pkgc.go ================================================ // pkgc is a tool for generating wrappers for Go packages imported by Grumpy // programs. // // usage: pkgc PACKAGE // // Where PACKAGE is the full Go package name. Generated code is dumped to // stdout. Packages generated in this way can be imported by Grumpy programs // using string literal import syntax, e.g.: // // import "__go__/encoding/json" // // Or: // // from "__go__/time" import Duration package main import ( "bytes" "fmt" "go/constant" "go/importer" "go/types" "math" "os" "path" ) const packageTemplate = `package %[1]s import ( "grumpy" "reflect" mod %[2]q ) func fun(f *grumpy.Frame, _ []*grumpy.Object) (*grumpy.Object, *grumpy.BaseException) { %[3]s return nil, nil } var Code = grumpy.NewCode("", %[2]q, nil, 0, fun) func init() { grumpy.RegisterModule("__go__/%[2]s", Code) } ` const typeTemplate = ` if true { var x mod.%[1]s if o, raised := grumpy.WrapNative(f, reflect.ValueOf(x)); raised != nil { return nil, raised } else if raised = f.Globals().SetItemString(f, %[1]q, o.Type().ToObject()); raised != nil { return nil, raised } } ` const varTemplate = ` if o, raised := grumpy.WrapNative(f, reflect.ValueOf(%[1]s)); raised != nil { return nil, raised } else if raised = f.Globals().SetItemString(f, %[2]q, o); raised != nil { return nil, raised } ` func getConst(name string, v constant.Value) string { format := "%s" switch v.Kind() { case constant.Int: if constant.Sign(v) >= 0 { if i, exact := constant.Uint64Val(v); exact { if i > math.MaxInt64 { format = "uint64(%s)" } } else { format = "float64(%s)" } } case constant.Float: format = "float64(%s)" } return fmt.Sprintf(format, name) } func main() { if len(os.Args) != 2 { fmt.Fprint(os.Stderr, "usage: pkgc PACKAGE") os.Exit(1) } pkgPath := os.Args[1] pkg, err := importer.Default().Import(pkgPath) if err != nil { fmt.Fprintf(os.Stderr, "failed to import: %q: %v\n", pkgPath, err) os.Exit(2) } var buf bytes.Buffer scope := pkg.Scope() for _, name := range scope.Names() { o := scope.Lookup(name) if !o.Exported() { continue } switch x := o.(type) { case *types.TypeName: if types.IsInterface(x.Type()) { continue } buf.WriteString(fmt.Sprintf(typeTemplate, name)) case *types.Const: expr := getConst("mod." + name, x.Val()) buf.WriteString(fmt.Sprintf(varTemplate, expr, name)) default: expr := "mod." + name buf.WriteString(fmt.Sprintf(varTemplate, expr, name)) } } fmt.Printf(packageTemplate, path.Base(pkgPath), pkgPath, buf.Bytes()) } ================================================ FILE: tools/pydeps ================================================ #!/usr/bin/env python # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Outputs names of modules imported by a script.""" import argparse import os import sys from grumpy.compiler import imputil from grumpy.compiler import util parser = argparse.ArgumentParser() parser.add_argument('script', help='Python source filename') parser.add_argument('-modname', default='__main__', help='Python module name') def main(args): gopath = os.getenv('GOPATH', None) if not gopath: print >> sys.stderr, 'GOPATH not set' return 1 try: imports = imputil.collect_imports(args.modname, args.script, gopath) except SyntaxError as e: print >> sys.stderr, '{}: line {}: invalid syntax: {}'.format( e.filename, e.lineno, e.text) return 2 except util.CompileError as e: print >> sys.stderr, str(e) return 2 names = set([args.modname]) for imp in imports: if imp.is_native: print imp.name else: parts = imp.name.split('.') # Iterate over all packages and the leaf module. for i in xrange(len(parts)): name = '.'.join(parts[:i+1]) if name not in names: names.add(name) print name if __name__ == '__main__': main(parser.parse_args())