Repository: ned14/pcpp Branch: master Commit: 988cc2f5eab4 Files: 210 Total size: 861.8 KB Directory structure: gitextract_sih0q66g/ ├── .gitattributes ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ └── ci.yml ├── .gitignore ├── .gitmodules ├── AGENTS.md ├── LICENSE.txt ├── README.rst ├── doc/ │ ├── evaluator.html │ ├── index.html │ ├── lextab.html │ ├── parser.html │ ├── parsetab.html │ ├── pcmd.html │ └── preprocessor.html ├── pcpp/ │ ├── __init__.py │ ├── evaluator.py │ ├── lextab.py │ ├── parser.py │ ├── parsetab.py │ ├── pcmd.py │ └── preprocessor.py ├── pyproject.toml ├── requirements.txt ├── setup.py └── tests/ ├── Readme.md ├── __init__.py ├── alternate_input_encodings.py ├── alternate_input_encodings1_ucs16le.c ├── alternate_input_encodings2_ucs16le.c ├── cstd.py ├── doctests.py ├── embedded.py ├── eval.py ├── issue0017/ │ ├── inc.h │ └── issue0017.c ├── issue0017-ref.i ├── issue0017.py ├── issue0025/ │ ├── inc.h │ └── main.c ├── issue0025-ref.i ├── issue0025.py ├── issue0027.py ├── issue0030/ │ ├── source1.c │ ├── source2.c │ └── source3.c ├── issue0030.py ├── issue0032.py ├── issue0037/ │ └── inc.h ├── issue0037.py ├── issue0044.h ├── issue0044.py ├── issue0051.c ├── issue0051.h ├── issue0051.py ├── issue0057.h ├── issue0057.py ├── issue0059.py ├── issue0059a.h ├── issue0059b.h ├── issue0063.c ├── issue0063.h ├── issue0063.py ├── issue0079.py ├── issue0098/ │ ├── dir1/ │ │ └── header.h │ ├── dir2/ │ │ └── header.h │ ├── dir3/ │ │ └── header.h │ └── dir4/ │ └── header.h ├── issue0098.py ├── issue0103.py ├── n_std-clang.i ├── n_std-gcc.i ├── n_std-pcpp.i ├── n_std.i ├── n_std.py ├── passthru.py └── test-c/ ├── LICENSE ├── defs.h ├── e_12_8.c ├── e_14.c ├── e_14_10.c ├── e_14_7.c ├── e_14_9.c ├── e_15_3.c ├── e_16.c ├── e_17.c ├── e_18_4.c ├── e_19_3.c ├── e_23_3.c ├── e_24_6.c ├── e_25_6.c ├── e_27_7.c ├── e_29_3.c ├── e_31.c ├── e_31_3.c ├── e_32_5.c ├── e_33_2.c ├── e_35_2.c ├── e_4_3.c ├── e_7_4.c ├── e_std.c ├── header.h ├── i_32_3.c ├── i_35.c ├── i_35_3.c ├── ifdef15.h ├── line.h ├── m1024.h ├── m_33_big5.c ├── m_33_eucjp.c ├── m_33_gb.c ├── m_33_jis.c ├── m_33_ksc.c ├── m_33_sjis.c ├── m_33_utf8.c ├── m_34_big5.c ├── m_34_eucjp.c ├── m_34_gb.c ├── m_34_jis.c ├── m_34_ksc.c ├── m_34_sjis.c ├── m_34_utf8.c ├── m_36_big5.c ├── m_36_jis.c ├── m_36_sjis.c ├── n_1.c ├── n_10.c ├── n_11.c ├── n_12.c ├── n_13.c ├── n_13_13.c ├── n_13_5.c ├── n_13_7.c ├── n_13_8.c ├── n_15.c ├── n_18.c ├── n_19.c ├── n_2.c ├── n_20.c ├── n_21.c ├── n_22.c ├── n_23.c ├── n_24.c ├── n_25.c ├── n_26.c ├── n_27.c ├── n_28.c ├── n_29.c ├── n_3.c ├── n_30.c ├── n_32.c ├── n_37.c ├── n_3_4.c ├── n_4.c ├── n_5.c ├── n_6.c ├── n_7.c ├── n_8.c ├── n_8_2.c ├── n_9.c ├── n_i_.lst ├── n_std.c ├── nest1.h ├── nest10.h ├── nest11.h ├── nest12.h ├── nest13.h ├── nest14.h ├── nest15.h ├── nest2.h ├── nest3.h ├── nest4.h ├── nest5.h ├── nest6.h ├── nest7.h ├── nest8.h ├── nest9.h ├── side_cpp ├── u_1_1.c ├── u_1_11.c ├── u_1_12.c ├── u_1_13.c ├── u_1_14.c ├── u_1_17.c ├── u_1_19.c ├── u_1_22.c ├── u_1_23.c ├── u_1_24.c ├── u_1_25.c ├── u_1_27.c ├── u_1_28.c ├── u_1_5.c ├── u_1_7_big5.c ├── u_1_7_eucjp.c ├── u_1_7_gb.c ├── u_1_7_jis.c ├── u_1_7_ksc.c ├── u_1_7_sjis.c ├── u_1_7_utf8.c ├── u_1_8.c ├── u_2.c ├── unbal1.h ├── unbal2.h ├── unbal3.h ├── unbal4.h ├── unbal5.h ├── unbal6.h ├── undefs.c ├── unspcs.c └── warns.c ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitattributes ================================================ * text=auto !eol svneol=native#text/plain *.gitattributes text svneol=native#text/plain # Scriptish formats *.bat text svneol=native#text/plain *.bsh text svneol=native#text/x-beanshell *.cgi text svneol=native#text/plain *.cmd text svneol=native#text/plain *.js text svneol=native#text/javascript *.php text svneol=native#text/x-php *.pl text svneol=native#text/x-perl *.pm text svneol=native#text/x-perl *.py text svneol=native#text/x-python *.sh eol=lf svneol=LF#text/x-sh configure eol=lf svneol=LF#text/x-sh # Image formats *.bmp binary svneol=unset#image/bmp *.gif binary svneol=unset#image/gif *.ico binary svneol=unset#image/ico *.jpeg binary svneol=unset#image/jpeg *.jpg binary svneol=unset#image/jpeg *.png binary svneol=unset#image/png *.tif binary svneol=unset#image/tiff *.tiff binary svneol=unset#image/tiff *.svg text svneol=native#image/svg%2Bxml # Data formats *.pdf binary svneol=unset#application/pdf *.avi binary svneol=unset#video/avi *.doc binary svneol=unset#application/msword *.dsp text svneol=crlf#text/plain *.dsw text svneol=crlf#text/plain *.eps binary svneol=unset#application/postscript *.json text svneol=native#application/json *.gz binary svneol=unset#application/gzip *.mov binary svneol=unset#video/quicktime *.mp3 binary svneol=unset#audio/mpeg *.ppt binary svneol=unset#application/vnd.ms-powerpoint *.ps binary svneol=unset#application/postscript *.psd binary svneol=unset#application/photoshop *.rdf binary svneol=unset#text/rdf *.rss text svneol=unset#text/xml *.rtf binary svneol=unset#text/rtf *.sln text svneol=native#text/plain *.swf binary svneol=unset#application/x-shockwave-flash *.tgz binary svneol=unset#application/gzip *.vcproj text svneol=native#text/xml *.vcxproj text svneol=native#text/xml *.vsprops text svneol=native#text/xml *.wav binary svneol=unset#audio/wav *.xls binary svneol=unset#application/vnd.ms-excel *.zip binary svneol=unset#application/zip # Text formats .htaccess text svneol=native#text/plain *.bbk text svneol=native#text/xml *.cmake text svneol=native#text/plain *.css text svneol=native#text/css *.dtd text svneol=native#text/xml *.htm text svneol=native#text/html *.html text svneol=native#text/html *.ini text svneol=native#text/plain *.log text svneol=native#text/plain *.mak text svneol=native#text/plain *.qbk text svneol=native#text/plain *.rst text svneol=native#text/plain *.sql text svneol=native#text/x-sql *.txt text svneol=native#text/plain *.xhtml text svneol=native#text/xhtml%2Bxml *.xml text svneol=native#text/xml *.xsd text svneol=native#text/xml *.xsl text svneol=native#text/xml *.xslt text svneol=native#text/xml *.xul text svneol=native#text/xul *.yml text svneol=native#text/plain boost-no-inspect text svneol=native#text/plain CHANGES text svneol=native#text/plain COPYING text svneol=native#text/plain INSTALL text svneol=native#text/plain Jamfile text svneol=native#text/plain Jamroot text svneol=native#text/plain Jamfile.v2 text svneol=native#text/plain Jamrules text svneol=native#text/plain Makefile* text svneol=native#text/plain README text svneol=native#text/plain TODO text svneol=native#text/plain # Code formats *.c text svneol=native#text/plain *.cpp text svneol=native#text/plain *.h text svneol=native#text/plain *.hpp text svneol=native#text/plain *.ipp text svneol=native#text/plain *.tpp text svneol=native#text/plain *.jam text svneol=native#text/plain *.java text svneol=native#text/plain ================================================ FILE: .github/FUNDING.yml ================================================ github: ned14 ================================================ FILE: .github/workflows/ci.yml ================================================ name: CI on: push: branches: - master pull_request: jobs: Build: runs-on: ubuntu-24.04 strategy: matrix: python-version: ['3.x', 'pypy3.11'] steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - shell: bash run: | pip install -r requirements.txt git submodule update --init --recursive - name: Build shell: bash run: | python setup.py build Test: runs-on: ubuntu-24.04 strategy: matrix: python-version: ['3.x', 'pypy3.11'] steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - shell: bash run: | pip install -r requirements.txt git submodule update --init --recursive - name: Test shell: bash run: | pip install pytest python -m pytest tests/ -v Install-Pip: runs-on: ubuntu-24.04 strategy: matrix: python-version: ['3.x', 'pypy3.11'] steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - shell: bash run: | pip install -r requirements.txt git submodule update --init --recursive - name: Install with pip shell: bash run: | python setup.py install pcpp --version Install-Uv: runs-on: ubuntu-24.04 strategy: matrix: python-version: ['3.x', 'pypy3.11'] steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install uv shell: bash run: | curl -LsSf https://astral.sh/uv/install.sh | sh echo "$HOME/.cargo/bin" >> $GITHUB_PATH - shell: bash run: | git submodule update --init --recursive - name: Install with uv shell: bash run: | uv venv test_env source test_env/bin/activate uv pip install -e . pcpp --version Test-Uv: runs-on: ubuntu-24.04 strategy: matrix: python-version: ['3.x', 'pypy3.11'] steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install uv shell: bash run: | curl -LsSf https://astral.sh/uv/install.sh | sh echo "$HOME/.cargo/bin" >> $GITHUB_PATH - shell: bash run: | git submodule update --init --recursive - name: Install dependencies and pcpp with uv shell: bash run: | uv venv test_env source test_env/bin/activate uv pip install -e . uv pip install pytest python -m pytest tests/ -v ================================================ FILE: .gitignore ================================================ *.pyc build/* dist/* pcpp.egg-info/* ================================================ FILE: .gitmodules ================================================ [submodule "pcpp/ply"] path = pcpp/ply url = https://github.com/ned14/ply.git branch = master ignore = untracked ================================================ FILE: AGENTS.md ================================================ # Agent Overview for pcpp (Pure Python C Preprocessor) ## How to build - `python setup.py build` ## How to test - `python setup.py test` ## Instructions 1. Ignore everything within the `ply` submodule. 2. Run tests before making a change. 3. Run tests after making a change. 4. If writing a unit test, always use the `unittest` framework. ================================================ FILE: LICENSE.txt ================================================ (C) 2018-2026 Niall Douglas http://www.nedproductions.biz/ and (C) 2007-2019 David Beazley http://www.dabeaz.com/ All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the David Beazley or Dabeaz LLC may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: README.rst ================================================ A C99 preprocessor written in pure Python ========================================= .. role:: c(code) :language: c .. |travis| image:: https://github.com/ned14/pcpp/actions/workflows/ci.yml/badge.svg?branch=master :align: middle :target: https://github.com/ned14/pcpp/actions \(C) 2018-2026 Niall Douglas http://www.nedproductions.biz/ and (C) 2007-2020 David Beazley http://www.dabeaz.com/ PyPI: https://pypi.python.org/pypi/pcpp Github: https://github.com/ned14/pcpp API reference docs: https://ned14.github.io/pcpp/ Travis master branch all tests passing for Python 3 and PyPy3: |travis| A pure universal Python C (pre-)preprocessor implementation very useful for pre-preprocessing header only C++ libraries into single file includes and other such build or packaging stage malarky. The implementation can be used as a Python module (`see API reference `_) or as a command line tool ``pcpp`` which can stand in for a conventional C preprocessor (i.e. it'll accept similar arguments). Works great under PyPy, and you can expect performance gains of between 0.84x and 2.62x (average = 2.2x, median = 2.31x). To install pcpp, you can use either pip or uv: Using pip: :: pip install pcpp Using uv (faster installation): :: uv install pcpp Your includes can be benchmarked for heft in order to improve your build times! See the ``--times`` and ``--filetimes`` options, and you can see graphs from pcpp for the C++ STLs at https://github.com/ned14/stl-header-heft. A very unique facility of this C preprocessor is *partial* preprocessing so you can programmatically control how much preprocessing is done by ``pcpp`` and how much is done by the C or C++ compiler's preprocessor. The ultimate control is by subclassing the :c:`Preprocessor` class in Python from which you can do anything you like, however for your convenience the ``pcpp`` command line tool comes with the following canned partial preprocessing algorithms: **passthru-defines** Pass through but still execute #defines and #undefs if not always removed by preprocessor logic. This ensures that including the output sets exactly the same macros as if you included the original, plus include guards work. **passthru-unfound-includes** If an :c:`#include` is not found, pass it through unmodified. This is very useful for passing through includes of system headers. **passthru-undefined-exprs** This is one of the most powerful pass through algorithms. If an expression passed to :c:`#if` (or its brethern) contains an unknown macro, expand the expression with known macros and pass through *unexecuted*, and then pass through the remaining block. Each :c:`#elif` is evaluated in turn and if it does not contain unknown macros, it will be executed immediately. Finally, any :c:`#else` clause is always passed through *unexecuted*. Note that include guards normally defeat this algorithm, so those are specially detected and ignored. **passthru-comments** A major use case for ``pcpp`` is as a preprocessor for the `doxygen `_ reference documentation tool whose preprocessor is unable to handle any preprocessing of any complexity. ``pcpp`` can partially execute the preprocessing which doxygen is incapable of, thus generating output which produces good results with doxygen. Hence the ability to pass through comments containing doxygen markup is very useful. **passthru-magic-macros** Don't expand ``__DATE__``, ``__TIME__``, ``__FILE__``, ``__LINE__`` nor ``__COUNTER__``. **passthru-includes** Don't expand those ``#include`` whose arguments match the supplied regular expression into the output, however still execute those includes. This lets you generate output with macros from nested includes expanded, however those ``#include`` matching the regular expression are passed through into the output. Standards (non-)compliance -------------------------- ``pcpp`` passes a very slightly modified edition of the `mcpp `_ unit test suite. The only modifications done were to disable the digraph and trigraphs tests. It also passes the list of "preprocessor torture" expansion fragments in the C11 standard, correctly expanding some very complex recursive macro expansions where expansions cause new macro expansions to be formed. In this, it handily beats the MSVC preprocessor and ought to handle most C99 preprocessor metaprogramming. If you compare its output side-by-side to that of GCC or clang's preprocessor, results are extremely close indeed with blank line collapsing being the only difference. As of v1.30 (Oct 2020), a proper yacc based expression evaluator for :c:`#if` expressions is used which is standards conforming, and fixes a large number of problems found in the previous Python :c:`eval()` based expression evaluator. A full, detailed list of known non-conformance with the C99 standard is below. Pull requests with bug fixes and new unit tests for the fix are welcome. On Python 3, input and output files can have your choice of encoding, and you can hook file open to inspect the encoding using ``chardet``. Note that most of this preprocessor was written originally by David Beazley to show off his excellent Python Lex-Yacc library PLY (http://www.dabeaz.com/ply/) and is hidden in there without being at all obvious given the number of Stack Overflow questions which have asked for a pure Python C preprocessor implementation. This implementation fixes a lot of conformance bugs (the original was never intended to rigidly adhere to the C standard) and adds in a test suite based on the C11 preprocessor torture samples plus the mcpp preprocessor test suite. Still, this project would not be possible without David's work, so please take off your hat and give a bow towards him. Command line tool ``pcpp``: --------------------------- The help from the command line tool ``pcpp``:: usage: pcpp [-h] [-o [path]] [-D macro[=val]] [-U macro] [-N macro] [-I path] [--passthru-defines] [--passthru-unfound-includes] [--passthru-unknown-exprs] [--passthru-comments] [--passthru-magic-macros] [--passthru-includes ] [--disable-auto-pragma-once] [--line-directive [form]] [--debug] [--time] [--filetimes [path]] [--compress] [--assume-input-encoding ] [--output-encoding ] [--write-bom] [--version] [input [input ...]] A pure universal Python C (pre-)preprocessor implementation very useful for pre-preprocessing header only C++ libraries into single file includes and other such build or packaging stage malarky. positional arguments: input Files to preprocess (use '-' for stdin) optional arguments: -h, --help show this help message and exit -o [path] Output to a file instead of stdout -D macro[=val] Predefine name as a macro [with value] -U macro Pre-undefine name as a macro -N macro Never define name as a macro, even if defined during the preprocessing. -I path Path to search for unfound #include's --passthru-defines Pass through but still execute #defines and #undefs if not always removed by preprocessor logic --passthru-unfound-includes Pass through #includes not found without execution --passthru-unknown-exprs Unknown macros in expressions cause preprocessor logic to be passed through instead of executed by treating unknown macros as 0L --passthru-comments Pass through comments unmodified --passthru-magic-macros Pass through double underscore magic macros unmodified --passthru-includes Regular expression for which #includes to not expand. #includes, if found, are always executed --disable-auto-pragma-once Disable the heuristics which auto apply #pragma once to #include files wholly wrapped in an obvious include guard macro --line-directive [form] Form of line directive to use, defaults to #line, specify nothing to disable output of line directives --debug Generate a pcpp_debug.log file logging execution --time Print the time it took to #include each file --filetimes [path] Write CSV file with time spent inside each included file, inclusive and exclusive --compress Make output as small as possible --assume-input-encoding The text encoding to assume inputs are in --output-encoding The text encoding to use when writing files --write-bom Prefix any output with a Unicode BOM --version show program's version number and exit Note that so pcpp can stand in for other preprocessor tooling, it ignores any arguments it does not understand. Quick demo of pass through mode ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Let us look at an example for pass through mode. Here is the original: .. code-block:: c #if !defined(__cpp_constexpr) #if __cplusplus >= 201402L #define __cpp_constexpr 201304 // relaxed constexpr #else #define __cpp_constexpr 190000 #endif #endif #ifndef BOOSTLITE_CONSTEXPR #if __cpp_constexpr >= 201304 #define BOOSTLITE_CONSTEXPR constexpr #endif #endif #ifndef BOOSTLITE_CONSTEXPR #define BOOSTLITE_CONSTEXPR #endif ``pcpp test.h --passthru-defines --passthru-unknown-exprs`` will output: .. code-block:: c #if !defined(__cpp_constexpr) #if __cplusplus >= 201402 #define __cpp_constexpr 201304 #else #define __cpp_constexpr 190000 #endif #endif #ifndef BOOSTLITE_CONSTEXPR #if __cpp_constexpr >= 201304 #define BOOSTLITE_CONSTEXPR constexpr #endif #endif #ifndef BOOSTLITE_CONSTEXPR #define BOOSTLITE_CONSTEXPR #endif This is because ``__cpp_constexpr`` was not defined, so because of the ``--passthru-unknown-exprs`` flag we pass through everything inside that if block **unexecuted** i.e. defines and undefs are NOT executed by ``pcpp``. Let's define ``__cpp_constexpr``: ``pcpp test.h --passthru-defines --passthru-unknown-exprs -D __cpp_constexpr`` .. code-block:: c #line 8 "test.h" #ifndef BOOSTLITE_CONSTEXPR #endif #ifndef BOOSTLITE_CONSTEXPR #define BOOSTLITE_CONSTEXPR #endif So, big difference now. We execute the entire first if block as ``__cpp_constexpr`` is now defined, thus leaving whitespace. Let's try setting ``__cpp_constexpr`` a bit higher: ``pcpp test.h --passthru-defines --passthru-unknown-exprs -D __cpp_constexpr=201304`` .. code-block:: c #line 8 "test.h" #ifndef BOOSTLITE_CONSTEXPR #define BOOSTLITE_CONSTEXPR constexpr #endif As you can see, the lines related to the known ``__cpp_constexpr`` are executed and removed, passing through any if blocks with unknown macros in the expression. What if you want a macro to be known but undefined? The -U (to undefine) flag has an obvious meaning in pass through mode in that it makes a macro no longer unknown, but known to be undefined. ``pcpp test.h --passthru-defines --passthru-unknown-exprs -U __cpp_constexpr`` .. code-block:: c #if __cplusplus >= 201402 #define __cpp_constexpr 201304 #else #define __cpp_constexpr 190000 #endif #ifndef BOOSTLITE_CONSTEXPR #endif #ifndef BOOSTLITE_CONSTEXPR #define BOOSTLITE_CONSTEXPR #endif Here ``__cpp_constexpr`` is known to be undefined so the first clause executes, but ``__cplusplus`` is unknown so that entire block is passed through unexecuted. In the next test comparing ``__cpp_constexpr`` to 201304 it is still known to be undefined, and so 0 >= 201304 is the expressions tested which is false, hence the following stanza is removed entirely. Helping ``pcpp`` using source code annotation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can achieve a great deal using -D (define), -U (undefine) and -N (never define) on the command line, but for more complex preprocessing it gets hard to pass through the correct logic without some source code annotation. ``pcpp`` lets you annotate which part of an if block being passed through due to use of unknown macros to also be executed in addition to the pass through. For this use ``__PCPP_ALWAYS_FALSE__`` or ``__PCPP_ALWAYS_TRUE__`` which tells ``pcpp`` to temporarily start executing the passed through preprocessor commands e.g. .. code-block:: c #if !defined(__cpp_constexpr) #if __cplusplus >= 201402L #define __cpp_constexpr 201304 #elif !__PCPP_ALWAYS_FALSE__ // pcpp please execute this next block #define __cpp_constexpr 190000 #endif #endif #ifndef BOOSTLITE_CONSTEXPR #if __cpp_constexpr >= 201304 #define BOOSTLITE_CONSTEXPR constexpr #endif #endif #ifndef BOOSTLITE_CONSTEXPR #define BOOSTLITE_CONSTEXPR #endif Note that ``__PCPP_ALWAYS_FALSE__`` will always be false in any other preprocessor, and it is also false in ``pcpp``. However, it causes ``pcpp`` to execute the define of ``__cpp_constexpr`` to 190000: ``pcpp test.h --passthru-defines --passthru-unknown-exprs`` .. code-block:: c #if !defined(__cpp_constexpr) #if __cplusplus >= 201402 #define __cpp_constexpr 201304 #elif 1 #define __cpp_constexpr 190000 #endif #endif #ifndef BOOSTLITE_CONSTEXPR #endif #ifndef BOOSTLITE_CONSTEXPR #define BOOSTLITE_CONSTEXPR #endif This is one way of marking up ``#else`` clauses so they always execute in a normal preprocessor and also pass through with execution with ``pcpp``. You can, of course, also place ``|| __PCPP_ALWAYS_FALSE__`` in any ``#if`` stanza to cause it to be passed through with execution, but not affect the preprocessing logic otherwise. What's implemented by the ``Preprocessor`` class: ================================================= - Digraphs and Trigraphs - line continuation operator '``\``' - C99 correct elimination of comments and maintenance of whitespace in output. - :c:`__DATE__`, :c:`__TIME__`, :c:`__FILE__`, :c:`__LINE__`. Note that :c:`__STDC__` et al are NOT defined by default, you need to define those manually before starting preprocessing. - :c:`__COUNTER__`, a very common extension - Object :c:`#define` - Function :c:`#define macro(...)` - Retokenisation and reexpansion after expansion is C99 compliant. - :c:`#undef` - :c:`#include "path"`, :c:`` and :c:`PATH` - :c:`defined` operator - C operators: - :c:`+, -, !, ~` - :c:`*, /, %` - :c:`+, -` - :c:`<<, >>` - :c:`<, <=, >, >=` - :c:`==, !=` - :c:`&` - :c:`^` - :c:`|` - :c:`&&` - :c:`||` - :c:`x ? y : z` (partial support, see known bugs) - :c:`#if`, :c:`#ifdef`, :c:`#ifndef`, :c:`#elif`, :c:`#else`, :c:`#endif` - Stringizing operator # - Token pasting operator ## - :c:`#pragma once`, a very common extension Additionally implemented by ``pcpp`` command line tool: ------------------------------------------------------- - :c:`#error` (default implementation prints to stderr and increments the exit code) - :c:`#warning` (default implementation prints to stderr) Not implemented yet (donations of code welcome): ------------------------------------------------ - :c:`#pragma` anything other than :c:`once`. - :c:`_Pragma` used to emit preprocessor calculated #pragma. - :c:`#line num`, :c:`num "file"` and :c:`NUMBER FILE`. Known bugs (ordered from worst to least worst): ----------------------------------------------- None presently known. Customising your own preprocessor: ================================== See the API reference docs at https://ned14.github.io/pcpp/ You can find an example of overriding the ``on_*()`` processing hooks at https://github.com/ned14/pcpp/blob/master/pcpp/pcmd.py Running Tests ============= To run the test suite for ``pcpp``, you can use either of these methods: 1. Using pytest directly (recommended): :: python -m pytest tests/ -v 2. Using the setup.py test command (deprecated but still functional): :: python setup.py test The test suite includes various test cases covering C99 preprocessor functionality, edge cases, and compatibility with the C11 standard preprocessor torture samples. History: ======== v1.31 (?): ---------- - Remove Python 2 support completely; pcpp is now Python 3 only (issue #87). - Replace setuptools test suite with pytest as the test runner. - Add ``uv`` support for faster dependency installation. - Rearrange ``main()`` function logic to avoid code duplication and make the entry point cleaner (PR #73). Thanks to assarbad for this improvement. - Fix issue #79 by replacing ``CPP_INTEGER`` and ``CPP_FLOAT`` tokens with a ``PP_NUMBER`` token for better preprocessing compliance. Update ``PP_NUMBER`` regex definition to properly handle digit separators in numeric literals. Add new test file for issue0079. Thanks to willwray for the PR implementing these features. - Add support for ``#include_next``, though note it is gated behind the ``--enable-include-next`` command line option. Thanks to Dudeldu for the original PR #98. - Multi line and unicode character literals were not working by pure oversight. Fixed and thanks to geky for showing the issue in PR #103. - Add support for ``__has_include``, a long requested and oft requested feature (#53, #77, #97). - Disable the processing of trigraphs by default to match other C preprocessors. Now pass ``--trigraphs`` to enable them. Thanks to pmp-p for suggesting this #100. - Believe it or not, until now this caused an infinite loop: ``` #define FOO(x) x #define BAR FOO(BAR) BAR ``` This is fixed, which closes #72, #101 and possibly quite a few more open issues. Thanks to MatthewShao for originally reporting this. v1.30 (29th October 2021): -------------------------- - Thanks to a 5km limit covid lockdown in my country, a public holiday where we were supposed to be away meant I was stuck at home instead. I took the full day to finish the https://github.com/ned14/pcpp/tree/yacc_expression_evaluator branch which is a proper C preprocessor expression evaluator based on http://www.dabeaz.com/ply/ 's yacc module. This was a very long outstanding piece of work which had been in progress for nearly two years. It just needed a full day of my time to get it done, and now it is indeed done at long last. - BREAKING CHANGE: Thanks to the new expression evaluator, fix a long standing bug where unknown function macros in expressions were parsed as ``0(0)`` which obviously enough does not work. Fixing this changes how the ``on_unknown_macro_in_expr()`` hook works, and there is now an added ``on_unknown_macro_function_in_expr()`` hook. - Add a new passthru option ``--passthru-includes`` which enables selected ``#include`` to be passed through, in addition to being executed. Thanks to schra for suggesting this, including a PR. The original implementation had some subtle corner case bugs, thanks to trelau for reporting those. - Fix a token expansion ordering bug whereby if a function macro used the same macro in more than one argument, expansion in one argument evaluation caused overly eager expansion in later argument evaluations. This fix ought to fix pcpp's ability to parse Boost (untested). Thanks to joaquintides for reporting this. - Now that pcpp no longer ever calls ``eval()``, pcpp is PyPy compatible and is probably also compatible with Pyston (untested). Typical speedup is about 2.2x-2.3x, though it can also be slower occasionally for some inputs. PyPy compatibility is now being tested by CI to ensure it remains working going forth. - Fix internal preprocessor error and failure to insert newlines before ``#include`` caused by certain sequence of line continuations in a macro. Thanks to dslijepcevic for reporting this. v1.22 (19th October 2020): -------------------------- - Fix bug where outputting to stdout did not combine with anything which printed to stdout. Thanks to Fondesa for reporting this. - Fix extra newlines being inserted after a multiline comment. Thanks to virtuald for sending a PR fixing this. - Fix not being able to actually specify an empty line directive. Thanks to kuri65536 for sending a PR fixing this. - Update ply submodule to latest from trunk. - Emit line continuations as tokens, rather than collapsing lines during parsing. Thanks to MathieuDuponchelle for the pull request implementing this. - Enable parsing and emission of files in arbitrary text encodings. This is supported in Python 3 or later only. Thanks to MathieuDuponchelle for the suggestion. - Fix bad regex for parsing floats, so now floats are correctly tokenised. Thanks to LynnKirby for reporting this. - BREAKING CHANGE: Passthrough for ``#include MACRO`` was not supported. This was not intentional, and to fix it required modifying the ``on_include_not_found()`` customisation point which is a source breaking change. Thanks to schra for reporting this. v1.21 (30th September 2019): ---------------------------- - Fix bug where token pasting two numeric tokens did not yield a numeric token. Thanks to Sei-Lisa for reporting this. - BREAKING CHANGE: Paths emitted by pcpp into ``#line`` directives now are relative to the working directory of the process when ``Preprocessor`` is initialised. This includes added search paths - files included from those locations will be emitted with a sequence of ``../`` to relativise the path emitted. If no path exists between the working directory and the path of the file being emitted, an absolute path is emitted instead. If you wish to disable this new behaviour, or use different behaviour, you can customise the new `rewrite_paths` member variable of ``Preprocessor``. - Fix bug where ``__LINE__`` was expanding into the line number of its definition instead of its use. Thanks to Sei-Lisa for reporting this. - Add ``--passthru-magic-macros`` command line option. - BREAKING CHANGE: The ``PreprocessorHooks`` and ``OutputDirective`` interface has changed. One now must specify the kind of ``OutputDirective`` abort one wants, and one can now both ignore AND remove directives. ``on_directive_handle()`` and ``on_directive_unknown()`` now take an extra parameter ``precedingtoks``, these are the tokens from the ``#`` up to the directive. - Fix a corner case where ``FUNC(void)foo()`` expanded to ``voidfoo()`` and not ``void foo()`` which is a very common non-conforming extension of the C preprocessor. Thanks to OmegaDoom for reporting this. - Add tokens for all the C operators, to help implementation of an expression evaluator. - Updated embedded ply to HEAD (2019-04-25) - Fix ``#include`` not working if no ``-I`` parameters were supplied. Thanks to csm10495 for reporting this. v1.20 (7th January 2019): ------------------------- - Now supports character literals in expressions. Thanks to untaugh for the pull request adding this. - Stopped the default collapsing of whitespace in output, and made it optional via a new command line option ``--compress``. - Fixed extraneous whitespace in ``--passthru-comments`` caused by multiline comments. Thanks to p2k for reporting this. - Fixed bug where defining a macro via string did not set the source attribute in the token. Thanks to ZedThree for reporting this. - Stop triggering an exception when no arguments are supplied to pcpp. Thanks to virtuald for reporting this. - Rebase onto PLY latest from Dec 28th 2018 (https://github.com/dabeaz/ply/commit/a37e0839583d683d95e70ce1445c0063c7d4bd21). Latest PLY no longer works using pypi packaging, David wants people to include the source of PLY directly. pcpp does this via a git submodule, and has setuptools bundle the submodule. - Add a formal LICENSE.txt file, as requested by Sei-Lisa. - Fix failure to issue ``#line`` directive for first include file in a file. Thanks to Sei-Lisa for reporting this. v1.1 (19th June 2018): ---------------------- - Added the ``--times`` and ``--filetimes`` features. - Fix bug where macros containing operator `defined` were not being expanded properly. - Added the ability to accept multiple inputs, they are concatenated into the output. - Fix bug where lines beginning with `#` and no contents caused an internal preprocessor error. - Fix bug where the macro expansion ``par par##ext`` was expanding into ``parext parext``. v1.01 (21st Feb 2018): ---------------------- - Fix bug where in pass through mode, an #elif in an #if block inside an #if block in ifpassthru was failing to be passed through. - Downgraded failure to evaluate an expression to a warning. - Fix missing Readme.rst in pypi package. v1.00 (13th Mar 2017): ---------------------- First release ================================================ FILE: doc/evaluator.html ================================================ pcpp.evaluator API documentation

pcpp.evaluator module

Source code
#!/usr/bin/python
# Python C99 conforming preprocessor expression evaluator
# (C) 2019-2026 Niall Douglas http://www.nedproductions.biz/
# Started: Apr 2019

from __future__ import generators, print_function, absolute_import, division

import sys, os, re, codecs, copy
if __name__ == '__main__' and __package__ is None:
    sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) )
from pcpp.parser import STRING_TYPES, yacc, default_lexer, in_production

# The width of signed integer which this evaluator will use
INTMAXBITS = 64

# Some Python 3 compatibility shims
if sys.version_info.major < 3:
    INTBASETYPE = long
else:
    INTBASETYPE = int

# Precompile the regular expression for correctly expanding unicode escape
# sequences in Python 2 and 3. See https://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python
# for more information.
_expand_escape_sequences_pat = re.compile(r'''
    ( \\U........      # 8-digit hex escapes
    | \\u....          # 4-digit hex escapes
    | \\x..            # 2-digit hex escapes
    | \\[0-7]{1,3}     # Octal escapes
    | \\N\{[^}]+\}     # Unicode characters by name
    | \\[\\'"abfnrtv]  # Single-character escapes
)''', re.UNICODE | re.VERBOSE)

class Value(INTBASETYPE):
    """A signed or unsigned integer within a preprocessor expression, bounded
    to within INT_MIN and INT_MAX, or 0 and UINT_MAX. Signed overflow is handled
    like a two's complement CPU, despite being UB, as that's what GCC and clang do.
    
    >>> Value(5)
    Value(5)
    >>> Value('5L')
    Value(5)
    >>> Value('5U')
    Value(5U)
    >>> Value('0')
    Value(0)
    >>> Value('0U')
    Value(0U)
    >>> Value('-1U')
    Value(18446744073709551615U)
    >>> Value(5) * Value(2)
    Value(10)
    >>> Value(5) + Value('2u')
    Value(7U)
    >>> Value(5) * 2
    Value(10)
    >>> Value(5) / 2   # Must return integer
    Value(2)
    >>> Value(50) % 8
    Value(2)
    >>> -Value(5)
    Value(-5)
    >>> +Value(-5)
    Value(-5)
    >>> ~Value(5)
    Value(-6)
    >>> Value(6) & 2
    Value(2)
    >>> Value(4) | 2
    Value(6)
    >>> Value(6) ^ 2
    Value(4)
    >>> Value(2) << 2
    Value(8)
    >>> Value(8) >> 2
    Value(2)
    >>> Value(9223372036854775808)
    Value(-9223372036854775808)
    >>> Value(-9223372036854775809)
    Value(9223372036854775807)
    >>> Value(18446744073709551615)
    Value(-1)
    >>> Value(False)
    Value(0)
    >>> Value(True)
    Value(1)
    >>> Value(5) == Value(6)
    Value(0)
    >>> Value(5) == Value(5)
    Value(1)
    >>> not Value(2)
    Traceback (most recent call last):
    ...
    AssertionError
    >>> Value(4) and Value(2)
    Traceback (most recent call last):
    ...
    AssertionError
    >>> Value(5) and not Value(6)
    Traceback (most recent call last):
    ...
    AssertionError
    >>> Value('0x3f')
    Value(63)
    >>> Value('077')
    Value(63)
    >>> Value("'N'")
    Value(78)
    >>> Value("L'N'")
    Value(78)
    >>> Value("'\\n'")
    Value(10)
    >>> Value("'\\\\n'")
    Value(10)
    >>> Value("'\\\\'")
    Value(92)
    >>> Value("'\\'")
    Traceback (most recent call last):
    ...
    SyntaxError: Empty character escape sequence
    """
    INT_MIN = -(1 << (INTMAXBITS - 1))
    INT_MAX = (1 << (INTMAXBITS - 1)) - 1
    INT_MASK = (1 << INTMAXBITS) - 1
    UINT_MIN = 0
    UINT_MAX = (1 << INTMAXBITS) - 1
    @classmethod
    def __sclamp(cls, value):
        value = INTBASETYPE(value)
        return ((value - cls.INT_MIN) & cls.INT_MASK) + cls.INT_MIN
    @classmethod
    def __uclamp(cls, value):
        value = INTBASETYPE(value)
        return value & cls.UINT_MAX
    def __new__(cls, value, unsigned = False, exception = None):
        if isinstance(value, Value):
            unsigned = value.unsigned
            exception = value.exception
        elif isinstance(value, INTBASETYPE) or isinstance(value, int) or isinstance(value, float):
            value = cls.__uclamp(value) if unsigned else cls.__sclamp(value)
        elif isinstance(value, STRING_TYPES):
            if (value.startswith("L'") or value[0] == "'") and value[-1] == "'":
                startidx = 2 if value.startswith("L'") else 1
                #print("1. ***", value, file = sys.stderr)
                value = value[startidx:-1]
                if len(value) == 0:
                    raise SyntaxError('Empty character escape sequence')
                #print("2. ***", value, file = sys.stderr)
                value = _expand_escape_sequences_pat.sub(lambda x: codecs.decode(x.group(0), 'unicode-escape'), value)
                #print("3. ***", value, file = sys.stderr)
                x = INTBASETYPE(ord(value))
                #print("4. ***", x, file = sys.stderr)
            elif value.startswith('0x') or value.startswith('0X'):
                # Strip any terminators
                while not ((value[-1] >= '0' and value[-1] <= '9') or (value[-1] >= 'a' and value[-1] <= 'f') or (value[-1] >= 'A' and value[-1] <= 'F')):
                    if value[-1] == 'u' or value[-1] == 'U':
                        unsigned = True
                    value = value[:-1]
                x = INTBASETYPE(value, base = 16)
            elif value.startswith('0'):
                # Strip any terminators
                while not (value[-1] >= '0' and value[-1] <= '7'):
                    if value[-1] == 'u' or value[-1] == 'U':
                        unsigned = True
                    value = value[:-1]
                x = INTBASETYPE(value, base = 8)
            else:
                # Strip any terminators
                while not (value[-1] >= '0' and value[-1] <= '9'):
                    if value[-1] == 'u' or value[-1] == 'U':
                        unsigned = True
                    value = value[:-1]
                x = INTBASETYPE(value)
            value = cls.__uclamp(x) if unsigned else cls.__sclamp(x)
            #assert x == value
        else:
            print('Unknown value type: %s' % repr(type(value)), file = sys.stderr)
            assert False  # Input is an unrecognised type
        inst = super(Value, cls).__new__(cls, value)
        inst.unsigned = unsigned
        inst.exception = exception
        return inst
    def value(self):
        if self.exception is not None:
            raise self.exception
        return INTBASETYPE(self)
    def __add__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) + self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__add__(other))
    def __sub__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) - self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__sub__(other))
    def __mul__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) * self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__mul__(other))
    def __div__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) / self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__div__(other))
    def __truediv__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) / self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__truediv__(other))
    def __mod__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) % self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__mod__(other))
    def __neg__(self):
        if self.exception is not None:
            return self
        return Value(super(Value, self).__neg__(), self.unsigned)
    def __invert__(self):
        if self.exception is not None:
            return self
        return Value(super(Value, self).__invert__(), self.unsigned)
    def __and__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) & self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__and__(other))
    def __or__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) | self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__or__(other))
    def __pos__(self):
        if self.exception is not None:
            return self
        return Value(super(Value, self).__pos__())
    def __pow__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) ** self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__pow__(other))
    def __lshift__(self, other):
        if self.exception is not None:
            return self
        # Ignore other signedness
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) << self.__uclamp(other), True) if (self.unsigned) else Value(super(Value, self).__lshift__(other))
    def __rshift__(self, other):
        if self.exception is not None:
            return self
        # Ignore other signedness
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) >> self.__uclamp(other), True) if (self.unsigned) else Value(super(Value, self).__rshift__(other))
    def __xor__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) ^ self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__xor__(other))
    def __repr__(self):
        if self.exception is not None:
            return "Exception(%s)" % repr(self.exception)
        elif self.unsigned:
            return "Value(%dU)" % INTBASETYPE(self)
        else:
            return "Value(%d)" % INTBASETYPE(self)
    def __bool__(self):
        assert False  # Do not use Python logical operations
    def __nonzero__(self):
        assert False  # Do not use Python logical operations
    def __cmp__(self, other):
        assert False
    def __lt__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) < self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) < self.__sclamp(other), False)
    def __le__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) <= self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) <= self.__sclamp(other), False)
    def __eq__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) == self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) == self.__sclamp(other), False)
    def __ne__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) != self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) != self.__sclamp(other), False)
    def __ge__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) >= self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) >= self.__sclamp(other), False)
    def __gt__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) > self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) > self.__sclamp(other), False)

        
# PLY yacc specification
# Valid C preprocessor expression items:
#   - Integer constants
#   - Character constants
#   - Addition, subtraction, multiplication, division, bitwise and-or-xor, shifts,
#     comparisons, logical and-or-not
#   - defined()
#
# The C preprocessor does not support:
#   - assignment
#   - increment and decrement
#   - array indexing, indirection
#   - casting
#   - sizeof, alignof

# The subset of tokens from Preprocessor used in preprocessor expressions
tokens = (
   'CPP_ID', 'CPP_INTEGER', 'CPP_CHAR', 'CPP_STRING',
   'CPP_PLUS', 'CPP_MINUS', 'CPP_STAR', 'CPP_FSLASH', 'CPP_PERCENT', 'CPP_BAR',
   'CPP_AMPERSAND', 'CPP_TILDE', 'CPP_HAT', 'CPP_LESS', 'CPP_GREATER', 'CPP_EXCLAMATION',
   'CPP_QUESTION', 'CPP_LPAREN', 'CPP_RPAREN',
   'CPP_COMMA', 'CPP_COLON',

   'CPP_LSHIFT', 'CPP_LESSEQUAL', 'CPP_RSHIFT',
   'CPP_GREATEREQUAL', 'CPP_LOGICALOR', 'CPP_LOGICALAND', 'CPP_EQUALITY',
   'CPP_INEQUALITY'
)
# 'CPP_WS', 'CPP_EQUAL',  'CPP_BSLASH', 'CPP_SQUOTE',

precedence = (
    ('left', 'CPP_COMMA'),                                                     # 15
                                                                               # 14 (assignments, unused)
    ('left', 'CPP_QUESTION', 'CPP_COLON'),                                     # 13
    ('left', 'CPP_LOGICALOR'),                                                 # 12
    ('left', 'CPP_LOGICALAND'),                                                # 11
    ('left', 'CPP_BAR'),                                                       # 10
    ('left', 'CPP_HAT'),                                                       # 9
    ('left', 'CPP_AMPERSAND'),                                                 # 8
    ('left', 'CPP_EQUALITY', 'CPP_INEQUALITY'),                                # 7
    ('left', 'CPP_LESS', 'CPP_LESSEQUAL', 'CPP_GREATER', 'CPP_GREATEREQUAL'),  # 6
    ('left', 'CPP_LSHIFT', 'CPP_RSHIFT'),                                      # 5
    ('left', 'CPP_PLUS', 'CPP_MINUS'),                                         # 4
    ('left', 'CPP_STAR', 'CPP_FSLASH', 'CPP_PERCENT'),                         # 3
    ('right', 'UPLUS', 'UMINUS', 'CPP_EXCLAMATION', 'CPP_TILDE'),              # 2
                                                                               # 1 (unused in the C preprocessor)
)

def p_error(p):
    if p:
        raise SyntaxError("around token '%s' type %s" % (p.value, p.type))
    else:
        raise SyntaxError("at EOF")

def p_expression_number(p):
    'expression : CPP_INTEGER'
    p[0] = Value(p[1])

def p_expression_character(p):
    'expression : CPP_CHAR'
    p[0] = Value(p[1])

def p_expression_string(p):
    """
    expression : CPP_STRING
              | CPP_LESS expression CPP_GREATER
    """
    p[0] = p[1]

def p_expression_group(t):
    'expression : CPP_LPAREN expression CPP_RPAREN'
    t[0] = t[2]

def p_expression_uplus(p):
    'expression : CPP_PLUS expression %prec UPLUS'
    p[0] = +Value(p[2])

def p_expression_uminus(p):
    'expression : CPP_MINUS expression %prec UMINUS'
    p[0] = -Value(p[2])

def p_expression_unop(p):
    """
    expression : CPP_EXCLAMATION expression
              | CPP_TILDE expression
    """
    try:
        if p[1] == '!':
            p[0] = Value(0) if (Value(p[2]).value()!=0) else Value(1)
        elif p[1] == '~':
            p[0] = ~Value(p[2])
    except Exception as e:
        p[0] = Value(0, exception = e)

def p_expression_binop(p):
    """
    expression : expression CPP_STAR expression
              | expression CPP_FSLASH expression
              | expression CPP_PERCENT expression
              | expression CPP_PLUS expression
              | expression CPP_MINUS expression
              | expression CPP_LSHIFT expression
              | expression CPP_RSHIFT expression
              | expression CPP_LESS expression
              | expression CPP_LESSEQUAL expression
              | expression CPP_GREATER expression
              | expression CPP_GREATEREQUAL expression
              | expression CPP_EQUALITY expression
              | expression CPP_INEQUALITY expression
              | expression CPP_AMPERSAND expression
              | expression CPP_HAT expression
              | expression CPP_BAR expression
              | expression CPP_LOGICALAND expression
              | expression CPP_LOGICALOR expression
              | expression CPP_COMMA expression
    """
    # print [repr(p[i]) for i in range(0,4)]
    try:
        if p[2] == '*':
            p[0] = Value(p[1]) * Value(p[3])
        elif p[2] == '/':
            p[0] = Value(p[1]) / Value(p[3])
        elif p[2] == '%':
            p[0] = Value(p[1]) % Value(p[3])
        elif p[2] == '+':
            p[0] = Value(p[1]) + Value(p[3])
        elif p[2] == '-':
            p[0] = Value(p[1]) - Value(p[3])
        elif p[2] == '<<':
            p[0] = Value(p[1]) << Value(p[3])
        elif p[2] == '>>':
            p[0] = Value(p[1]) >> Value(p[3])
        elif p[2] == '<':
            p[0] = Value(p[1]) < Value(p[3])
        elif p[2] == '<=':
            p[0] = Value(p[1]) <= Value(p[3])
        elif p[2] == '>':
            p[0] = Value(p[1]) > Value(p[3])
        elif p[2] == '>=':
            p[0] = Value(p[1]) >= Value(p[3])
        elif p[2] == '==':
            p[0] = Value(p[1]) == Value(p[3])
        elif p[2] == '!=':
            p[0] = Value(p[1]) != Value(p[3])
        elif p[2] == '&':
            p[0] = Value(p[1]) & Value(p[3])
        elif p[2] == '^':
            p[0] = Value(p[1]) ^ Value(p[3])
        elif p[2] == '|':
            p[0] = Value(p[1]) | Value(p[3])
        elif p[2] == '&&':
            p[0] = Value(1) if (Value(p[1]).value()!=0 and Value(p[3]).value()!=0) else Value(0)
        elif p[2] == '||':
            p[0] = Value(1) if (Value(p[1]).value()!=0 or Value(p[3]).value()!=0) else Value(0)
        elif p[2] == ',':
            p[0] = Value(p[3])
    except Exception as e:
        p[0] = Value(0, exception = e)

def p_expression_conditional(p):
    'expression : expression CPP_QUESTION expression CPP_COLON expression'
    try:
        # Output type must cast up to unsigned if either input is unsigned
        p[0] = Value(p[3]) if (Value(p[1]).value()!=0) else Value(p[5])
        try:
            p[0] = Value(p[0].value(), unsigned = Value(p[3]).unsigned or Value(p[5]).unsigned)
        except:
            pass
    except Exception as e:
        p[0] = Value(0, exception = e)

def p_expression_function_call(p):
    "expression : CPP_ID CPP_LPAREN expression CPP_RPAREN"
    try:
        p.lexer.on_function_call(p)
    except Exception as e:
        p[0] = Value(0, exception = e)

def p_expression_identifier(p):
    "expression : CPP_ID"
    try:
        p.lexer.on_identifier(p)
    except Exception as e:
        p[0] = Value(0, exception = e)


class Evaluator(object):
    """Evaluator of #if C preprocessor expressions.
    
    >>> e = Evaluator()
    >>> e('5')
    Value(5)
    >>> e('5+6')
    Value(11)
    >>> e('5+6*2')
    Value(17)
    >>> e('5/2+6*2')
    Value(14)
    >>> e('5 < 6 <= 7')
    Value(1)
    >>> e('5 < 6 && 8 > 7')
    Value(1)
    >>> e('18446744073709551615 == -1')
    Value(1)
    >>> e('-9223372036854775809 == 9223372036854775807')
    Value(1)
    >>> e('-1 < 0U')
    Value(0U)
    >>> e('(( 0L && 0) || (!0L && !0 ))')
    Value(1)
    >>> e('(1)?2:3')
    Value(2)
    >>> e('(1 ? -1 : 0) <= 0')
    Value(1)
    >>> e('(1 ? -1 : 0U)')       # Output type of ? must be common between both choices
    Value(18446744073709551615U)
    >>> e('(1 ? -1 : 0U) <= 0')
    Value(0U)
    >>> e('1 && 10 / 0')         # doctest: +ELLIPSIS
    Exception(ZeroDivisionError('division by zero'...
    >>> e('0 && 10 / 0')         # && must shortcut
    Value(0)
    >>> e('1 ? 10 / 0 : 0')      # doctest: +ELLIPSIS
    Exception(ZeroDivisionError('division by zero'...
    >>> e('0 ? 10 / 0 : 0')      # ? must shortcut
    Value(0)
    >>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
    Value(0)
    >>> e('1 << 2 != 4 || 8 >> 1 != 4')
    Value(0)
    >>> e('(2 || 3) != 1 || (2 && 3) != 1 || (0 || 4) != 1 || (0 && 5) != 0')
    Value(0)
    >>> e('-1 << 3U > 0')
    Value(0)
    >>> e("'N' == 78")
    Value(1)
    >>> e('0x3f == 63')
    Value(1)
    >>> e("'\\\\n'")
    Value(10)
    >>> e("'\\\\\\\\'")
    Value(92)
    >>> e("'\\\\n' == 0xA")
    Value(1)
    >>> e("'\\\\\\\\' == 0x5c")
    Value(1)
    >>> e("L'\\\\0' == 0")
    Value(1)
    >>> e('12 == 12')
    Value(1)
    >>> e('12L == 12')
    Value(1)
    >>> e('-1 >= 0U')
    Value(1U)
    >>> e('(1<<2) == 4')
    Value(1)
    >>> e('(-!+!9) == -1')
    Value(1)
    >>> e('(2 || 3) == 1')
    Value(1)
    >>> e('1L * 3 != 3')
    Value(0)
    >>> e('(!1L != 0) || (-1L != -1)')
    Value(0)
    >>> e('0177777 == 65535')
    Value(1)
    >>> e('0Xffff != 65535 || 0XFfFf == 65535')
    Value(1)
    >>> e('0L != 0 || 0l != 0')
    Value(0)
    >>> e('1U != 1 || 1u == 1')
    Value(1)
    >>> e('0 <= -1')
    Value(0)
    >>> e('1 << 2 != 4 || 8 >> 1 == 4')
    Value(1)
    >>> e('(3 ^ 5) == 6')
    Value(1)
    >>> e('(3 | 5) == 7')
    Value(1)
    >>> e('(3 & 5) == 1')
    Value(1)
    >>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
    Value(0)
    >>> e('(0 ? 1 : 2) != 2')
    Value(0)
    >>> e('-1 << 3U > 0')
    Value(0)
    >>> e('0 && 10 / 0')
    Value(0)
    >>> e('not_defined && 10 / not_defined')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown identifier not_defined'...
    >>> e('0 && 10 / 0 > 1')
    Value(0)
    >>> e('(0) ? 10 / 0 : 0')
    Value(0)
    >>> e('0 == 0 || 10 / 0 > 1')
    Value(1)
    >>> e('(15 >> 2 >> 1 != 1) || (3 << 2 << 1 != 24)')
    Value(0)
    >>> e('(1 | 2) == 3 && 4 != 5 || 0')
    Value(1)
    >>> e('1  >  0')
    Value(1)
    >>> e("'\123' != 83")
    Value(0)
    >>> e("'\x1b' != '\033'")
    Value(0)
    >>> e('0 + (1 - (2 + (3 - (4 + (5 - (6 + (7 - (8 + (9 - (10 + (11 - (12 +          (13 - (14 + (15 - (16 + (17 - (18 + (19 - (20 + (21 - (22 + (23 -           (24 + (25 - (26 + (27 - (28 + (29 - (30 + (31 - (32 + 0))))))))))           )))))))))))))))))))))) == 0')
    Value(1)
    >>> e('test_function(X)', functions={'test_function':lambda x: 55})
    Value(55)
    >>> e('test_identifier', identifiers={'test_identifier':11})
    Value(11)
    >>> e('defined(X)', functions={'defined':lambda x: 55})
    Value(55)
    >>> e('defined(X)')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown function defined'...
    >>> e('__has_include("variant")')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown function __has_include'...
    >>> e('__has_include(<variant>)')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown function __has_include'...
    >>> e('5  // comment')
    Value(5)
    >>> e('5  /* comment */')
    Value(5)
    >>> e('5  /* comment // more */')
    Value(5)
    >>> e('5  // /* comment */')
    Value(5)
    """
#    >>> e('defined X', functions={'defined':lambda x: 55})
#    Value(55)

    def __init__(self, lexer = None):
        self.lexer = lexer if lexer is not None else default_lexer()
        self.parser = yacc.yacc(optimize=in_production,debug=not in_production,write_tables=not in_production)

    class __lexer(object):

        def __init__(self, functions, identifiers):
            self.__toks = []
            self.__functions = functions
            self.__identifiers = identifiers

        def input(self, toks):
            self.__toks = [tok for tok in toks if tok.type != 'CPP_WS' and tok.type != 'CPP_LINECONT' and tok.type != 'CPP_COMMENT1' and tok.type != 'CPP_COMMENT2']
            self.__idx = 0

        def token(self):
            if self.__idx >= len(self.__toks):
                return None
            self.__idx = self.__idx + 1
            return self.__toks[self.__idx - 1]

        def on_function_call(self, p):
            if p[1] not in self.__functions:
                raise SyntaxError('Unknown function %s' % p[1])
            p[0] = Value(self.__functions[p[1]](p[3]))

        def on_identifier(self, p):
            if p[1] not in self.__identifiers:
                raise SyntaxError('Unknown identifier %s' % p[1])
            p[0] = Value(self.__identifiers[p[1]])
            
    def __call__(self, input, functions = {}, identifiers = {}):
        """Execute a fully macro expanded set of tokens representing an expression,
        returning the result of the evaluation.
        """
        if not isinstance(input,list):
            self.lexer.input(input)
            input = []
            while True:
                tok = self.lexer.token()
                if not tok:
                    break
                input.append(tok)
        return self.parser.parse(input, lexer = self.__lexer(functions, identifiers))


if __name__ == "__main__":
    import doctest
    doctest.testmod()

Functions

def p_error(p)
Source code
def p_error(p):
    if p:
        raise SyntaxError("around token '%s' type %s" % (p.value, p.type))
    else:
        raise SyntaxError("at EOF")
def p_expression_binop(p)
expression : expression CPP_STAR expression
| expression CPP_FSLASH expression | expression CPP_PERCENT expression | expression CPP_PLUS expression | expression CPP_MINUS expression | expression CPP_LSHIFT expression | expression CPP_RSHIFT expression | expression CPP_LESS expression | expression CPP_LESSEQUAL expression | expression CPP_GREATER expression | expression CPP_GREATEREQUAL expression | expression CPP_EQUALITY expression | expression CPP_INEQUALITY expression | expression CPP_AMPERSAND expression | expression CPP_HAT expression | expression CPP_BAR expression | expression CPP_LOGICALAND expression | expression CPP_LOGICALOR expression | expression CPP_COMMA expression
Source code
def p_expression_binop(p):
    """
    expression : expression CPP_STAR expression
              | expression CPP_FSLASH expression
              | expression CPP_PERCENT expression
              | expression CPP_PLUS expression
              | expression CPP_MINUS expression
              | expression CPP_LSHIFT expression
              | expression CPP_RSHIFT expression
              | expression CPP_LESS expression
              | expression CPP_LESSEQUAL expression
              | expression CPP_GREATER expression
              | expression CPP_GREATEREQUAL expression
              | expression CPP_EQUALITY expression
              | expression CPP_INEQUALITY expression
              | expression CPP_AMPERSAND expression
              | expression CPP_HAT expression
              | expression CPP_BAR expression
              | expression CPP_LOGICALAND expression
              | expression CPP_LOGICALOR expression
              | expression CPP_COMMA expression
    """
    # print [repr(p[i]) for i in range(0,4)]
    try:
        if p[2] == '*':
            p[0] = Value(p[1]) * Value(p[3])
        elif p[2] == '/':
            p[0] = Value(p[1]) / Value(p[3])
        elif p[2] == '%':
            p[0] = Value(p[1]) % Value(p[3])
        elif p[2] == '+':
            p[0] = Value(p[1]) + Value(p[3])
        elif p[2] == '-':
            p[0] = Value(p[1]) - Value(p[3])
        elif p[2] == '<<':
            p[0] = Value(p[1]) << Value(p[3])
        elif p[2] == '>>':
            p[0] = Value(p[1]) >> Value(p[3])
        elif p[2] == '<':
            p[0] = Value(p[1]) < Value(p[3])
        elif p[2] == '<=':
            p[0] = Value(p[1]) <= Value(p[3])
        elif p[2] == '>':
            p[0] = Value(p[1]) > Value(p[3])
        elif p[2] == '>=':
            p[0] = Value(p[1]) >= Value(p[3])
        elif p[2] == '==':
            p[0] = Value(p[1]) == Value(p[3])
        elif p[2] == '!=':
            p[0] = Value(p[1]) != Value(p[3])
        elif p[2] == '&':
            p[0] = Value(p[1]) & Value(p[3])
        elif p[2] == '^':
            p[0] = Value(p[1]) ^ Value(p[3])
        elif p[2] == '|':
            p[0] = Value(p[1]) | Value(p[3])
        elif p[2] == '&&':
            p[0] = Value(1) if (Value(p[1]).value()!=0 and Value(p[3]).value()!=0) else Value(0)
        elif p[2] == '||':
            p[0] = Value(1) if (Value(p[1]).value()!=0 or Value(p[3]).value()!=0) else Value(0)
        elif p[2] == ',':
            p[0] = Value(p[3])
    except Exception as e:
        p[0] = Value(0, exception = e)
def p_expression_character(p)
expression : CPP_CHAR
 
Source code
def p_expression_character(p):
    'expression : CPP_CHAR'
    p[0] = Value(p[1])
def p_expression_conditional(p)
expression : expression CPP_QUESTION expression CPP_COLON expression
 
Source code
def p_expression_conditional(p):
    'expression : expression CPP_QUESTION expression CPP_COLON expression'
    try:
        # Output type must cast up to unsigned if either input is unsigned
        p[0] = Value(p[3]) if (Value(p[1]).value()!=0) else Value(p[5])
        try:
            p[0] = Value(p[0].value(), unsigned = Value(p[3]).unsigned or Value(p[5]).unsigned)
        except:
            pass
    except Exception as e:
        p[0] = Value(0, exception = e)
def p_expression_function_call(p)
expression : CPP_ID CPP_LPAREN expression CPP_RPAREN
 
Source code
def p_expression_function_call(p):
    "expression : CPP_ID CPP_LPAREN expression CPP_RPAREN"
    try:
        p.lexer.on_function_call(p)
    except Exception as e:
        p[0] = Value(0, exception = e)
def p_expression_group(t)
expression : CPP_LPAREN expression CPP_RPAREN
 
Source code
def p_expression_group(t):
    'expression : CPP_LPAREN expression CPP_RPAREN'
    t[0] = t[2]
def p_expression_identifier(p)
expression : CPP_ID
 
Source code
def p_expression_identifier(p):
    "expression : CPP_ID"
    try:
        p.lexer.on_identifier(p)
    except Exception as e:
        p[0] = Value(0, exception = e)
def p_expression_number(p)
expression : CPP_INTEGER
 
Source code
def p_expression_number(p):
    'expression : CPP_INTEGER'
    p[0] = Value(p[1])
def p_expression_string(p)
expression : CPP_STRING
| CPP_LESS expression CPP_GREATER
Source code
def p_expression_string(p):
    """
    expression : CPP_STRING
              | CPP_LESS expression CPP_GREATER
    """
    p[0] = p[1]
def p_expression_uminus(p)
expression : CPP_MINUS expression %prec UMINUS
 
Source code
def p_expression_uminus(p):
    'expression : CPP_MINUS expression %prec UMINUS'
    p[0] = -Value(p[2])
def p_expression_unop(p)
expression : CPP_EXCLAMATION expression
| CPP_TILDE expression
Source code
def p_expression_unop(p):
    """
    expression : CPP_EXCLAMATION expression
              | CPP_TILDE expression
    """
    try:
        if p[1] == '!':
            p[0] = Value(0) if (Value(p[2]).value()!=0) else Value(1)
        elif p[1] == '~':
            p[0] = ~Value(p[2])
    except Exception as e:
        p[0] = Value(0, exception = e)
def p_expression_uplus(p)
expression : CPP_PLUS expression %prec UPLUS
 
Source code
def p_expression_uplus(p):
    'expression : CPP_PLUS expression %prec UPLUS'
    p[0] = +Value(p[2])

Classes

class Evaluator

Evaluator of #if C preprocessor expressions.

>>> e = Evaluator()
>>> e('5')
Value(5)
>>> e('5+6')
Value(11)
>>> e('5+6*2')
Value(17)
>>> e('5/2+6*2')
Value(14)
>>> e('5 < 6 <= 7')
Value(1)
>>> e('5 < 6 && 8 > 7')
Value(1)
>>> e('18446744073709551615 == -1')
Value(1)
>>> e('-9223372036854775809 == 9223372036854775807')
Value(1)
>>> e('-1 < 0U')
Value(0U)
>>> e('(( 0L && 0) || (!0L && !0 ))')
Value(1)
>>> e('(1)?2:3')
Value(2)
>>> e('(1 ? -1 : 0) <= 0')
Value(1)
>>> e('(1 ? -1 : 0U)')       # Output type of ? must be common between both choices
Value(18446744073709551615U)
>>> e('(1 ? -1 : 0U) <= 0')
Value(0U)
>>> e('1 && 10 / 0')         # doctest: +ELLIPSIS
Exception(ZeroDivisionError('division by zero'...
>>> e('0 && 10 / 0')         # && must shortcut
Value(0)
>>> e('1 ? 10 / 0 : 0')      # doctest: +ELLIPSIS
Exception(ZeroDivisionError('division by zero'...
>>> e('0 ? 10 / 0 : 0')      # ? must shortcut
Value(0)
>>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
Value(0)
>>> e('1 << 2 != 4 || 8 >> 1 != 4')
Value(0)
>>> e('(2 || 3) != 1 || (2 && 3) != 1 || (0 || 4) != 1 || (0 && 5) != 0')
Value(0)
>>> e('-1 << 3U > 0')
Value(0)
>>> e("'N' == 78")
Value(1)
>>> e('0x3f == 63')
Value(1)
>>> e("'\\n'")
Value(10)
>>> e("'\\\\'")
Value(92)
>>> e("'\\n' == 0xA")
Value(1)
>>> e("'\\\\' == 0x5c")
Value(1)
>>> e("L'\\0' == 0")
Value(1)
>>> e('12 == 12')
Value(1)
>>> e('12L == 12')
Value(1)
>>> e('-1 >= 0U')
Value(1U)
>>> e('(1<<2) == 4')
Value(1)
>>> e('(-!+!9) == -1')
Value(1)
>>> e('(2 || 3) == 1')
Value(1)
>>> e('1L * 3 != 3')
Value(0)
>>> e('(!1L != 0) || (-1L != -1)')
Value(0)
>>> e('0177777 == 65535')
Value(1)
>>> e('0Xffff != 65535 || 0XFfFf == 65535')
Value(1)
>>> e('0L != 0 || 0l != 0')
Value(0)
>>> e('1U != 1 || 1u == 1')
Value(1)
>>> e('0 <= -1')
Value(0)
>>> e('1 << 2 != 4 || 8 >> 1 == 4')
Value(1)
>>> e('(3 ^ 5) == 6')
Value(1)
>>> e('(3 | 5) == 7')
Value(1)
>>> e('(3 & 5) == 1')
Value(1)
>>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
Value(0)
>>> e('(0 ? 1 : 2) != 2')
Value(0)
>>> e('-1 << 3U > 0')
Value(0)
>>> e('0 && 10 / 0')
Value(0)
>>> e('not_defined && 10 / not_defined')  # doctest: +ELLIPSIS
Exception(SyntaxError('Unknown identifier not_defined'...
>>> e('0 && 10 / 0 > 1')
Value(0)
>>> e('(0) ? 10 / 0 : 0')
Value(0)
>>> e('0 == 0 || 10 / 0 > 1')
Value(1)
>>> e('(15 >> 2 >> 1 != 1) || (3 << 2 << 1 != 24)')
Value(0)
>>> e('(1 | 2) == 3 && 4 != 5 || 0')
Value(1)
>>> e('1  >  0')
Value(1)
>>> e("'S' != 83")
Value(0)
>>> e("'' != ''")
Value(0)
>>> e('0 + (1 - (2 + (3 - (4 + (5 - (6 + (7 - (8 + (9 - (10 + (11 - (12 +          (13 - (14 + (15 - (16 + (17 - (18 + (19 - (20 + (21 - (22 + (23 -           (24 + (25 - (26 + (27 - (28 + (29 - (30 + (31 - (32 + 0))))))))))           )))))))))))))))))))))) == 0')
Value(1)
>>> e('test_function(X)', functions={'test_function':lambda x: 55})
Value(55)
>>> e('test_identifier', identifiers={'test_identifier':11})
Value(11)
>>> e('defined(X)', functions={'defined':lambda x: 55})
Value(55)
>>> e('defined(X)')  # doctest: +ELLIPSIS
Exception(SyntaxError('Unknown function defined'...
>>> e('__has_include("variant")')  # doctest: +ELLIPSIS
Exception(SyntaxError('Unknown function __has_include'...
>>> e('__has_include(<variant>)')  # doctest: +ELLIPSIS
Exception(SyntaxError('Unknown function __has_include'...
>>> e('5  // comment')
Value(5)
>>> e('5  /* comment */')
Value(5)
>>> e('5  /* comment // more */')
Value(5)
>>> e('5  // /* comment */')

Value(5)

Source code
class Evaluator(object):
    """Evaluator of #if C preprocessor expressions.
    
    >>> e = Evaluator()
    >>> e('5')
    Value(5)
    >>> e('5+6')
    Value(11)
    >>> e('5+6*2')
    Value(17)
    >>> e('5/2+6*2')
    Value(14)
    >>> e('5 < 6 <= 7')
    Value(1)
    >>> e('5 < 6 && 8 > 7')
    Value(1)
    >>> e('18446744073709551615 == -1')
    Value(1)
    >>> e('-9223372036854775809 == 9223372036854775807')
    Value(1)
    >>> e('-1 < 0U')
    Value(0U)
    >>> e('(( 0L && 0) || (!0L && !0 ))')
    Value(1)
    >>> e('(1)?2:3')
    Value(2)
    >>> e('(1 ? -1 : 0) <= 0')
    Value(1)
    >>> e('(1 ? -1 : 0U)')       # Output type of ? must be common between both choices
    Value(18446744073709551615U)
    >>> e('(1 ? -1 : 0U) <= 0')
    Value(0U)
    >>> e('1 && 10 / 0')         # doctest: +ELLIPSIS
    Exception(ZeroDivisionError('division by zero'...
    >>> e('0 && 10 / 0')         # && must shortcut
    Value(0)
    >>> e('1 ? 10 / 0 : 0')      # doctest: +ELLIPSIS
    Exception(ZeroDivisionError('division by zero'...
    >>> e('0 ? 10 / 0 : 0')      # ? must shortcut
    Value(0)
    >>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
    Value(0)
    >>> e('1 << 2 != 4 || 8 >> 1 != 4')
    Value(0)
    >>> e('(2 || 3) != 1 || (2 && 3) != 1 || (0 || 4) != 1 || (0 && 5) != 0')
    Value(0)
    >>> e('-1 << 3U > 0')
    Value(0)
    >>> e("'N' == 78")
    Value(1)
    >>> e('0x3f == 63')
    Value(1)
    >>> e("'\\\\n'")
    Value(10)
    >>> e("'\\\\\\\\'")
    Value(92)
    >>> e("'\\\\n' == 0xA")
    Value(1)
    >>> e("'\\\\\\\\' == 0x5c")
    Value(1)
    >>> e("L'\\\\0' == 0")
    Value(1)
    >>> e('12 == 12')
    Value(1)
    >>> e('12L == 12')
    Value(1)
    >>> e('-1 >= 0U')
    Value(1U)
    >>> e('(1<<2) == 4')
    Value(1)
    >>> e('(-!+!9) == -1')
    Value(1)
    >>> e('(2 || 3) == 1')
    Value(1)
    >>> e('1L * 3 != 3')
    Value(0)
    >>> e('(!1L != 0) || (-1L != -1)')
    Value(0)
    >>> e('0177777 == 65535')
    Value(1)
    >>> e('0Xffff != 65535 || 0XFfFf == 65535')
    Value(1)
    >>> e('0L != 0 || 0l != 0')
    Value(0)
    >>> e('1U != 1 || 1u == 1')
    Value(1)
    >>> e('0 <= -1')
    Value(0)
    >>> e('1 << 2 != 4 || 8 >> 1 == 4')
    Value(1)
    >>> e('(3 ^ 5) == 6')
    Value(1)
    >>> e('(3 | 5) == 7')
    Value(1)
    >>> e('(3 & 5) == 1')
    Value(1)
    >>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
    Value(0)
    >>> e('(0 ? 1 : 2) != 2')
    Value(0)
    >>> e('-1 << 3U > 0')
    Value(0)
    >>> e('0 && 10 / 0')
    Value(0)
    >>> e('not_defined && 10 / not_defined')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown identifier not_defined'...
    >>> e('0 && 10 / 0 > 1')
    Value(0)
    >>> e('(0) ? 10 / 0 : 0')
    Value(0)
    >>> e('0 == 0 || 10 / 0 > 1')
    Value(1)
    >>> e('(15 >> 2 >> 1 != 1) || (3 << 2 << 1 != 24)')
    Value(0)
    >>> e('(1 | 2) == 3 && 4 != 5 || 0')
    Value(1)
    >>> e('1  >  0')
    Value(1)
    >>> e("'\123' != 83")
    Value(0)
    >>> e("'\x1b' != '\033'")
    Value(0)
    >>> e('0 + (1 - (2 + (3 - (4 + (5 - (6 + (7 - (8 + (9 - (10 + (11 - (12 +          (13 - (14 + (15 - (16 + (17 - (18 + (19 - (20 + (21 - (22 + (23 -           (24 + (25 - (26 + (27 - (28 + (29 - (30 + (31 - (32 + 0))))))))))           )))))))))))))))))))))) == 0')
    Value(1)
    >>> e('test_function(X)', functions={'test_function':lambda x: 55})
    Value(55)
    >>> e('test_identifier', identifiers={'test_identifier':11})
    Value(11)
    >>> e('defined(X)', functions={'defined':lambda x: 55})
    Value(55)
    >>> e('defined(X)')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown function defined'...
    >>> e('__has_include("variant")')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown function __has_include'...
    >>> e('__has_include(<variant>)')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown function __has_include'...
    >>> e('5  // comment')
    Value(5)
    >>> e('5  /* comment */')
    Value(5)
    >>> e('5  /* comment // more */')
    Value(5)
    >>> e('5  // /* comment */')
    Value(5)
    """
#    >>> e('defined X', functions={'defined':lambda x: 55})
#    Value(55)

    def __init__(self, lexer = None):
        self.lexer = lexer if lexer is not None else default_lexer()
        self.parser = yacc.yacc(optimize=in_production,debug=not in_production,write_tables=not in_production)

    class __lexer(object):

        def __init__(self, functions, identifiers):
            self.__toks = []
            self.__functions = functions
            self.__identifiers = identifiers

        def input(self, toks):
            self.__toks = [tok for tok in toks if tok.type != 'CPP_WS' and tok.type != 'CPP_LINECONT' and tok.type != 'CPP_COMMENT1' and tok.type != 'CPP_COMMENT2']
            self.__idx = 0

        def token(self):
            if self.__idx >= len(self.__toks):
                return None
            self.__idx = self.__idx + 1
            return self.__toks[self.__idx - 1]

        def on_function_call(self, p):
            if p[1] not in self.__functions:
                raise SyntaxError('Unknown function %s' % p[1])
            p[0] = Value(self.__functions[p[1]](p[3]))

        def on_identifier(self, p):
            if p[1] not in self.__identifiers:
                raise SyntaxError('Unknown identifier %s' % p[1])
            p[0] = Value(self.__identifiers[p[1]])
            
    def __call__(self, input, functions = {}, identifiers = {}):
        """Execute a fully macro expanded set of tokens representing an expression,
        returning the result of the evaluation.
        """
        if not isinstance(input,list):
            self.lexer.input(input)
            input = []
            while True:
                tok = self.lexer.token()
                if not tok:
                    break
                input.append(tok)
        return self.parser.parse(input, lexer = self.__lexer(functions, identifiers))

Methods

def __init__(self, lexer=None)

Initialize self. See help(type(self)) for accurate signature.

Source code
def __init__(self, lexer = None):
    self.lexer = lexer if lexer is not None else default_lexer()
    self.parser = yacc.yacc(optimize=in_production,debug=not in_production,write_tables=not in_production)
class Value (ancestors: builtins.int)

A signed or unsigned integer within a preprocessor expression, bounded to within INT_MIN and INT_MAX, or 0 and UINT_MAX. Signed overflow is handled like a two's complement CPU, despite being UB, as that's what GCC and clang do.

>>> Value(5)
Value(5)
>>> Value('5L')
Value(5)
>>> Value('5U')
Value(5U)
>>> Value('0')
Value(0)
>>> Value('0U')
Value(0U)
>>> Value('-1U')
Value(18446744073709551615U)
>>> Value(5) * Value(2)
Value(10)
>>> Value(5) + Value('2u')
Value(7U)
>>> Value(5) * 2
Value(10)
>>> Value(5) / 2   # Must return integer
Value(2)
>>> Value(50) % 8
Value(2)
>>> -Value(5)
Value(-5)
>>> +Value(-5)
Value(-5)
>>> ~Value(5)
Value(-6)
>>> Value(6) & 2
Value(2)
>>> Value(4) | 2
Value(6)
>>> Value(6) ^ 2
Value(4)
>>> Value(2) << 2
Value(8)
>>> Value(8) >> 2
Value(2)
>>> Value(9223372036854775808)
Value(-9223372036854775808)
>>> Value(-9223372036854775809)
Value(9223372036854775807)
>>> Value(18446744073709551615)
Value(-1)
>>> Value(False)
Value(0)
>>> Value(True)
Value(1)
>>> Value(5) == Value(6)
Value(0)
>>> Value(5) == Value(5)
Value(1)
>>> not Value(2)
Traceback (most recent call last):

… AssertionError

>>> Value(4) and Value(2)
Traceback (most recent call last):

… AssertionError

>>> Value(5) and not Value(6)
Traceback (most recent call last):

… AssertionError

>>> Value('0x3f')
Value(63)
>>> Value('077')
Value(63)
>>> Value("'N'")
Value(78)
>>> Value("L'N'")
Value(78)
>>> Value("'\n'")
Value(10)
>>> Value("'\\n'")
Value(10)
>>> Value("'\\'")
Value(92)
>>> Value("'\'")
Traceback (most recent call last):
SyntaxError : Empty character escape sequence
 
Source code
class Value(INTBASETYPE):
    """A signed or unsigned integer within a preprocessor expression, bounded
    to within INT_MIN and INT_MAX, or 0 and UINT_MAX. Signed overflow is handled
    like a two's complement CPU, despite being UB, as that's what GCC and clang do.
    
    >>> Value(5)
    Value(5)
    >>> Value('5L')
    Value(5)
    >>> Value('5U')
    Value(5U)
    >>> Value('0')
    Value(0)
    >>> Value('0U')
    Value(0U)
    >>> Value('-1U')
    Value(18446744073709551615U)
    >>> Value(5) * Value(2)
    Value(10)
    >>> Value(5) + Value('2u')
    Value(7U)
    >>> Value(5) * 2
    Value(10)
    >>> Value(5) / 2   # Must return integer
    Value(2)
    >>> Value(50) % 8
    Value(2)
    >>> -Value(5)
    Value(-5)
    >>> +Value(-5)
    Value(-5)
    >>> ~Value(5)
    Value(-6)
    >>> Value(6) & 2
    Value(2)
    >>> Value(4) | 2
    Value(6)
    >>> Value(6) ^ 2
    Value(4)
    >>> Value(2) << 2
    Value(8)
    >>> Value(8) >> 2
    Value(2)
    >>> Value(9223372036854775808)
    Value(-9223372036854775808)
    >>> Value(-9223372036854775809)
    Value(9223372036854775807)
    >>> Value(18446744073709551615)
    Value(-1)
    >>> Value(False)
    Value(0)
    >>> Value(True)
    Value(1)
    >>> Value(5) == Value(6)
    Value(0)
    >>> Value(5) == Value(5)
    Value(1)
    >>> not Value(2)
    Traceback (most recent call last):
    ...
    AssertionError
    >>> Value(4) and Value(2)
    Traceback (most recent call last):
    ...
    AssertionError
    >>> Value(5) and not Value(6)
    Traceback (most recent call last):
    ...
    AssertionError
    >>> Value('0x3f')
    Value(63)
    >>> Value('077')
    Value(63)
    >>> Value("'N'")
    Value(78)
    >>> Value("L'N'")
    Value(78)
    >>> Value("'\\n'")
    Value(10)
    >>> Value("'\\\\n'")
    Value(10)
    >>> Value("'\\\\'")
    Value(92)
    >>> Value("'\\'")
    Traceback (most recent call last):
    ...
    SyntaxError: Empty character escape sequence
    """
    INT_MIN = -(1 << (INTMAXBITS - 1))
    INT_MAX = (1 << (INTMAXBITS - 1)) - 1
    INT_MASK = (1 << INTMAXBITS) - 1
    UINT_MIN = 0
    UINT_MAX = (1 << INTMAXBITS) - 1
    @classmethod
    def __sclamp(cls, value):
        value = INTBASETYPE(value)
        return ((value - cls.INT_MIN) & cls.INT_MASK) + cls.INT_MIN
    @classmethod
    def __uclamp(cls, value):
        value = INTBASETYPE(value)
        return value & cls.UINT_MAX
    def __new__(cls, value, unsigned = False, exception = None):
        if isinstance(value, Value):
            unsigned = value.unsigned
            exception = value.exception
        elif isinstance(value, INTBASETYPE) or isinstance(value, int) or isinstance(value, float):
            value = cls.__uclamp(value) if unsigned else cls.__sclamp(value)
        elif isinstance(value, STRING_TYPES):
            if (value.startswith("L'") or value[0] == "'") and value[-1] == "'":
                startidx = 2 if value.startswith("L'") else 1
                #print("1. ***", value, file = sys.stderr)
                value = value[startidx:-1]
                if len(value) == 0:
                    raise SyntaxError('Empty character escape sequence')
                #print("2. ***", value, file = sys.stderr)
                value = _expand_escape_sequences_pat.sub(lambda x: codecs.decode(x.group(0), 'unicode-escape'), value)
                #print("3. ***", value, file = sys.stderr)
                x = INTBASETYPE(ord(value))
                #print("4. ***", x, file = sys.stderr)
            elif value.startswith('0x') or value.startswith('0X'):
                # Strip any terminators
                while not ((value[-1] >= '0' and value[-1] <= '9') or (value[-1] >= 'a' and value[-1] <= 'f') or (value[-1] >= 'A' and value[-1] <= 'F')):
                    if value[-1] == 'u' or value[-1] == 'U':
                        unsigned = True
                    value = value[:-1]
                x = INTBASETYPE(value, base = 16)
            elif value.startswith('0'):
                # Strip any terminators
                while not (value[-1] >= '0' and value[-1] <= '7'):
                    if value[-1] == 'u' or value[-1] == 'U':
                        unsigned = True
                    value = value[:-1]
                x = INTBASETYPE(value, base = 8)
            else:
                # Strip any terminators
                while not (value[-1] >= '0' and value[-1] <= '9'):
                    if value[-1] == 'u' or value[-1] == 'U':
                        unsigned = True
                    value = value[:-1]
                x = INTBASETYPE(value)
            value = cls.__uclamp(x) if unsigned else cls.__sclamp(x)
            #assert x == value
        else:
            print('Unknown value type: %s' % repr(type(value)), file = sys.stderr)
            assert False  # Input is an unrecognised type
        inst = super(Value, cls).__new__(cls, value)
        inst.unsigned = unsigned
        inst.exception = exception
        return inst
    def value(self):
        if self.exception is not None:
            raise self.exception
        return INTBASETYPE(self)
    def __add__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) + self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__add__(other))
    def __sub__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) - self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__sub__(other))
    def __mul__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) * self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__mul__(other))
    def __div__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) / self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__div__(other))
    def __truediv__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) / self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__truediv__(other))
    def __mod__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) % self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__mod__(other))
    def __neg__(self):
        if self.exception is not None:
            return self
        return Value(super(Value, self).__neg__(), self.unsigned)
    def __invert__(self):
        if self.exception is not None:
            return self
        return Value(super(Value, self).__invert__(), self.unsigned)
    def __and__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) & self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__and__(other))
    def __or__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) | self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__or__(other))
    def __pos__(self):
        if self.exception is not None:
            return self
        return Value(super(Value, self).__pos__())
    def __pow__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) ** self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__pow__(other))
    def __lshift__(self, other):
        if self.exception is not None:
            return self
        # Ignore other signedness
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) << self.__uclamp(other), True) if (self.unsigned) else Value(super(Value, self).__lshift__(other))
    def __rshift__(self, other):
        if self.exception is not None:
            return self
        # Ignore other signedness
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) >> self.__uclamp(other), True) if (self.unsigned) else Value(super(Value, self).__rshift__(other))
    def __xor__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) ^ self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__xor__(other))
    def __repr__(self):
        if self.exception is not None:
            return "Exception(%s)" % repr(self.exception)
        elif self.unsigned:
            return "Value(%dU)" % INTBASETYPE(self)
        else:
            return "Value(%d)" % INTBASETYPE(self)
    def __bool__(self):
        assert False  # Do not use Python logical operations
    def __nonzero__(self):
        assert False  # Do not use Python logical operations
    def __cmp__(self, other):
        assert False
    def __lt__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) < self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) < self.__sclamp(other), False)
    def __le__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) <= self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) <= self.__sclamp(other), False)
    def __eq__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) == self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) == self.__sclamp(other), False)
    def __ne__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) != self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) != self.__sclamp(other), False)
    def __ge__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) >= self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) >= self.__sclamp(other), False)
    def __gt__(self, other):
        if self.exception is not None:
            return self
        other = Value(other)
        if other.exception is not None:
            return other
        return Value(self.__uclamp(self) > self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) > self.__sclamp(other), False)

Class variables

var INT_MASK
var INT_MAX
var INT_MIN
var UINT_MAX
var UINT_MIN

Methods

def value(self)
Source code
def value(self):
    if self.exception is not None:
        raise self.exception
    return INTBASETYPE(self)
================================================ FILE: doc/index.html ================================================ pcpp API documentation
================================================ FILE: doc/lextab.html ================================================ pcpp.lextab API documentation

pcpp.lextab module

Source code
# lextab.py. This file automatically created by PLY (version 3.11). Don't edit!
_tabversion   = '3.10'
_lextokens    = set(('CPP_AMPERSAND', 'CPP_ANDEQUAL', 'CPP_BAR', 'CPP_BSLASH', 'CPP_CHAR', 'CPP_COLON', 'CPP_COMMA', 'CPP_COMMENT1', 'CPP_COMMENT2', 'CPP_DEREFERENCE', 'CPP_DIVIDEEQUAL', 'CPP_DOT', 'CPP_DPOUND', 'CPP_DQUOTE', 'CPP_EQUAL', 'CPP_EQUALITY', 'CPP_EXCLAMATION', 'CPP_FLOAT', 'CPP_FSLASH', 'CPP_GREATER', 'CPP_GREATEREQUAL', 'CPP_HAT', 'CPP_ID', 'CPP_INEQUALITY', 'CPP_INTEGER', 'CPP_LBRACKET', 'CPP_LCURLY', 'CPP_LESS', 'CPP_LESSEQUAL', 'CPP_LINECONT', 'CPP_LOGICALAND', 'CPP_LOGICALOR', 'CPP_LPAREN', 'CPP_LSHIFT', 'CPP_LSHIFTEQUAL', 'CPP_MINUS', 'CPP_MINUSEQUAL', 'CPP_MINUSMINUS', 'CPP_MULTIPLYEQUAL', 'CPP_OREQUAL', 'CPP_PERCENT', 'CPP_PERCENTEQUAL', 'CPP_PLUS', 'CPP_PLUSEQUAL', 'CPP_PLUSPLUS', 'CPP_POUND', 'CPP_QUESTION', 'CPP_RBRACKET', 'CPP_RCURLY', 'CPP_RPAREN', 'CPP_RSHIFT', 'CPP_RSHIFTEQUAL', 'CPP_SEMICOLON', 'CPP_SQUOTE', 'CPP_STAR', 'CPP_STRING', 'CPP_TILDE', 'CPP_WS', 'CPP_XOREQUAL'))
_lexreflags   = 64
_lexliterals  = '+-*/%|&~^<>=!?()[]{}.,;:\\\'"'
_lexstateinfo = {'INITIAL': 'inclusive'}
_lexstatere   = {'INITIAL': [('(?P<t_CPP_WS>([ \\t]+|\\n))|(?P<t_CPP_LINECONT>\\\\[ \\t]*\\n)|(?P<t_CPP_INTEGER>(((((0x)|(0X))[0-9a-fA-F]+)|(\\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?))|(?P<t_CPP_STRING>\\"([^\\\\\\n]|(\\\\(.|\\n)))*?\\")|(?P<t_CPP_CHAR>(L)?\\\'([^\\\\\\n]|(\\\\(.|\\n)))*?\\\')|(?P<t_CPP_COMMENT1>(/\\*(.|\\n)*?\\*/))|(?P<t_CPP_COMMENT2>(//[^\\n]*))|(?P<t_CPP_FLOAT>((\\d+)(\\.\\d+)(e(\\+|-)?(\\d+))?|(\\d+)e(\\+|-)?(\\d+))([lL]|[fF])?)|(?P<t_CPP_ID>[A-Za-z_][\\w_]*)|(?P<t_CPP_LOGICALOR>\\|\\|)|(?P<t_CPP_PLUSPLUS>\\+\\+)|(?P<t_CPP_DPOUND>\\#\\#)|(?P<t_CPP_LSHIFTEQUAL><<=)|(?P<t_CPP_OREQUAL>\\|=)|(?P<t_CPP_PLUSEQUAL>\\+=)|(?P<t_CPP_RSHIFTEQUAL>>>=)|(?P<t_CPP_MULTIPLYEQUAL>\\*=)|(?P<t_CPP_BAR>\\|)|(?P<t_CPP_DIVIDEEQUAL>/=)|(?P<t_CPP_POUND>\\#)|(?P<t_CPP_PERCENTEQUAL>%=)|(?P<t_CPP_DEREFERENCE>->)|(?P<t_CPP_RPAREN>\\))|(?P<t_CPP_ANDEQUAL>&=)|(?P<t_CPP_RBRACKET>\\])|(?P<t_CPP_LPAREN>\\()|(?P<t_CPP_RSHIFT>>>)|(?P<t_CPP_LESSEQUAL><=)|(?P<t_CPP_HAT>\\^)|(?P<t_CPP_LOGICALAND>&&)|(?P<t_CPP_EQUALITY>==)|(?P<t_CPP_GREATEREQUAL>>=)|(?P<t_CPP_BSLASH>\\\\)|(?P<t_CPP_MINUSEQUAL>-=)|(?P<t_CPP_DOT>\\.)|(?P<t_CPP_MINUSMINUS>--)|(?P<t_CPP_LBRACKET>\\[)|(?P<t_CPP_PLUS>\\+)|(?P<t_CPP_XOREQUAL>^=)|(?P<t_CPP_STAR>\\*)|(?P<t_CPP_QUESTION>\\?)|(?P<t_CPP_LSHIFT><<)|(?P<t_CPP_INEQUALITY>!=)|(?P<t_CPP_DQUOTE>")|(?P<t_CPP_MINUS>-)|(?P<t_CPP_RCURLY>})|(?P<t_CPP_GREATER>>)|(?P<t_CPP_LESS><)|(?P<t_CPP_SQUOTE>\')|(?P<t_CPP_EXCLAMATION>!)|(?P<t_CPP_LCURLY>{)|(?P<t_CPP_EQUAL>=)|(?P<t_CPP_FSLASH>/)|(?P<t_CPP_COLON>:)|(?P<t_CPP_AMPERSAND>&)|(?P<t_CPP_COMMA>,)|(?P<t_CPP_TILDE>~)|(?P<t_CPP_SEMICOLON>;)|(?P<t_CPP_PERCENT>%)', [None, ('t_CPP_WS', 'CPP_WS'), None, ('t_CPP_LINECONT', 'CPP_LINECONT'), ('t_CPP_INTEGER', 'CPP_INTEGER'), None, None, None, None, None, None, None, None, ('t_CPP_STRING', 'CPP_STRING'), None, None, None, ('t_CPP_CHAR', 'CPP_CHAR'), None, None, None, None, ('t_CPP_COMMENT1', 'CPP_COMMENT1'), None, None, ('t_CPP_COMMENT2', 'CPP_COMMENT2'), None, (None, 'CPP_FLOAT'), None, None, None, None, None, None, None, None, None, None, (None, 'CPP_ID'), (None, 'CPP_LOGICALOR'), (None, 'CPP_PLUSPLUS'), (None, 'CPP_DPOUND'), (None, 'CPP_LSHIFTEQUAL'), (None, 'CPP_OREQUAL'), (None, 'CPP_PLUSEQUAL'), (None, 'CPP_RSHIFTEQUAL'), (None, 'CPP_MULTIPLYEQUAL'), (None, 'CPP_BAR'), (None, 'CPP_DIVIDEEQUAL'), (None, 'CPP_POUND'), (None, 'CPP_PERCENTEQUAL'), (None, 'CPP_DEREFERENCE'), (None, 'CPP_RPAREN'), (None, 'CPP_ANDEQUAL'), (None, 'CPP_RBRACKET'), (None, 'CPP_LPAREN'), (None, 'CPP_RSHIFT'), (None, 'CPP_LESSEQUAL'), (None, 'CPP_HAT'), (None, 'CPP_LOGICALAND'), (None, 'CPP_EQUALITY'), (None, 'CPP_GREATEREQUAL'), (None, 'CPP_BSLASH'), (None, 'CPP_MINUSEQUAL'), (None, 'CPP_DOT'), (None, 'CPP_MINUSMINUS'), (None, 'CPP_LBRACKET'), (None, 'CPP_PLUS'), (None, 'CPP_XOREQUAL'), (None, 'CPP_STAR'), (None, 'CPP_QUESTION'), (None, 'CPP_LSHIFT'), (None, 'CPP_INEQUALITY'), (None, 'CPP_DQUOTE'), (None, 'CPP_MINUS'), (None, 'CPP_RCURLY'), (None, 'CPP_GREATER'), (None, 'CPP_LESS'), (None, 'CPP_SQUOTE'), (None, 'CPP_EXCLAMATION'), (None, 'CPP_LCURLY'), (None, 'CPP_EQUAL'), (None, 'CPP_FSLASH'), (None, 'CPP_COLON'), (None, 'CPP_AMPERSAND'), (None, 'CPP_COMMA'), (None, 'CPP_TILDE'), (None, 'CPP_SEMICOLON'), (None, 'CPP_PERCENT')])]}
_lexstateignore = {'INITIAL': ''}
_lexstateerrorf = {'INITIAL': 't_error'}
_lexstateeoff = {}
================================================ FILE: doc/parser.html ================================================ pcpp.parser API documentation

pcpp.parser module

Source code
#!/usr/bin/python
# Python C99 conforming preprocessor parser config
# (C) 2017-2026 Niall Douglas http://www.nedproductions.biz/
# and (C) 2007-2017 David Beazley http://www.dabeaz.com/
# Started: Feb 2017
#
# This C preprocessor was originally written by David Beazley and the
# original can be found at https://github.com/dabeaz/ply/blob/master/ply/cpp.py
# This edition substantially improves on standards conforming output,
# getting quite close to what clang or GCC outputs.

from __future__ import generators, print_function, absolute_import, division

import sys, re, os

in_production = 1  # Set to 0 if editing pcpp implementation!

# Some Python 3 compatibility shims
if sys.version_info.major < 3:
    STRING_TYPES = (str, unicode)
else:
    STRING_TYPES = str

# -----------------------------------------------------------------------------
# Default preprocessor lexer definitions.   These tokens are enough to get
# a basic preprocessor working.   Other modules may import these if they want
# -----------------------------------------------------------------------------

tokens = (
   'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_LINECONT', 'CPP_COMMENT1', 'CPP_COMMENT2',
   'CPP_POUND','CPP_DPOUND', 'CPP_PLUS', 'CPP_MINUS', 'CPP_STAR', 'CPP_FSLASH', 'CPP_PERCENT', 'CPP_BAR',
   'CPP_AMPERSAND', 'CPP_TILDE', 'CPP_HAT', 'CPP_LESS', 'CPP_GREATER', 'CPP_EQUAL', 'CPP_EXCLAMATION',
   'CPP_QUESTION', 'CPP_LPAREN', 'CPP_RPAREN', 'CPP_LBRACKET', 'CPP_RBRACKET', 'CPP_LCURLY', 'CPP_RCURLY',
   'CPP_DOT', 'CPP_COMMA', 'CPP_SEMICOLON', 'CPP_COLON', 'CPP_BSLASH', 'CPP_SQUOTE', 'CPP_DQUOTE',

   'CPP_DEREFERENCE', 'CPP_MINUSEQUAL', 'CPP_MINUSMINUS', 'CPP_LSHIFT', 'CPP_LESSEQUAL', 'CPP_RSHIFT',
   'CPP_GREATEREQUAL', 'CPP_LOGICALOR', 'CPP_OREQUAL', 'CPP_LOGICALAND', 'CPP_ANDEQUAL', 'CPP_EQUALITY',
   'CPP_INEQUALITY', 'CPP_XOREQUAL', 'CPP_MULTIPLYEQUAL', 'CPP_DIVIDEEQUAL', 'CPP_PLUSEQUAL', 'CPP_PLUSPLUS',
   'CPP_PERCENTEQUAL', 'CPP_LSHIFTEQUAL', 'CPP_RSHIFTEQUAL'
)

literals = "+-*/%|&~^<>=!?()[]{}.,;:\\\'\""

# Whitespace, but don't match past the end of a line
def t_CPP_WS(t):
    r'([ \t]+|\n)'
    t.lexer.lineno += t.value.count("\n")
    return t

# Line continuation, accept whitespace between the backslash and new line
def t_CPP_LINECONT(t):
    r'\\[ \t]*\n'
    t.value = t.value[1:-1]
    t.lexer.lineno += 1
    return t
_string_literal_linecont_pat = re.compile(r'\\[ \t]*\n')

t_CPP_POUND = r'\#'
t_CPP_DPOUND = r'\#\#'
t_CPP_PLUS = r'\+'
t_CPP_MINUS = r'-'
t_CPP_STAR = r'\*'
t_CPP_FSLASH = r'/'
t_CPP_PERCENT = r'%'
t_CPP_BAR = r'\|'
t_CPP_AMPERSAND = r'&'
t_CPP_TILDE = r'~'
t_CPP_HAT = r'\^'
t_CPP_LESS = r'<'
t_CPP_GREATER = r'>'
t_CPP_EQUAL = r'='
t_CPP_EXCLAMATION = r'!'
t_CPP_QUESTION = r'\?'
t_CPP_LPAREN = r'\('
t_CPP_RPAREN = r'\)'
t_CPP_LBRACKET = r'\['
t_CPP_RBRACKET = r'\]'
t_CPP_LCURLY = r'{'
t_CPP_RCURLY = r'}'
t_CPP_DOT = r'\.'
t_CPP_COMMA = r','
t_CPP_SEMICOLON = r';'
t_CPP_COLON = r':'
t_CPP_BSLASH = r'\\'
t_CPP_SQUOTE = r"'"
t_CPP_DQUOTE = r'"'

t_CPP_DEREFERENCE = r'->'
t_CPP_MINUSEQUAL = r'-='
t_CPP_MINUSMINUS = r'--'
t_CPP_LSHIFT = r'<<'
t_CPP_LESSEQUAL = r'<='
t_CPP_RSHIFT = r'>>'
t_CPP_GREATEREQUAL = r'>='
t_CPP_LOGICALOR = r'\|\|'
t_CPP_OREQUAL = r'\|='
t_CPP_LOGICALAND = r'&&'
t_CPP_ANDEQUAL = r'&='
t_CPP_EQUALITY = r'=='
t_CPP_INEQUALITY = r'!='
t_CPP_XOREQUAL = r'^='
t_CPP_MULTIPLYEQUAL = r'\*='
t_CPP_DIVIDEEQUAL = r'/='
t_CPP_PLUSEQUAL = r'\+='
t_CPP_PLUSPLUS = r'\+\+'
t_CPP_PERCENTEQUAL = r'%='
t_CPP_LSHIFTEQUAL = r'<<='
t_CPP_RSHIFTEQUAL = r'>>='


# Identifier
t_CPP_ID = r'[A-Za-z_][\w_]*'

# Integer literal
def CPP_INTEGER(t):
    r'(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)'
    return t

t_CPP_INTEGER = CPP_INTEGER

# Floating literal
t_CPP_FLOAT = r'((\d+)(\.\d+)(e(\+|-)?(\d+))?|(\d+)e(\+|-)?(\d+))([lL]|[fF])?'

# String literal
def t_CPP_STRING(t):
    r'\"([^\\\n]|(\\(.|\n)))*?\"'
    t.value, subs_made = _string_literal_linecont_pat.subn('', t.value)
    t.lexer.lineno += subs_made + t.value.count("\n")
    return t

# Character constant 'c' or L'c'
def t_CPP_CHAR(t):
    r'(L)?\'([^\\\n]|(\\(.|\n)))*?\''
    t.lexer.lineno += t.value.count("\n")
    return t

# Comment
def t_CPP_COMMENT1(t):
    r'(/\*(.|\n)*?\*/)'
    ncr = t.value.count("\n")
    t.lexer.lineno += ncr
    return t

# Line comment
def t_CPP_COMMENT2(t):
    r'(//[^\n]*)'
    return t
    
def t_error(t):
    t.type = t.value[0]
    t.value = t.value[0]
    t.lexer.skip(1)
    return t


# Python 2/3 compatible way of importing a subpackage
oldsyspath = sys.path
sys.path = [ os.path.join( os.path.dirname( os.path.abspath(__file__) ), "ply" ) ] + sys.path
from ply import lex, yacc
from ply.lex import LexToken
sys.path = oldsyspath
del oldsyspath

# -----------------------------------------------------------------------------
# trigraph()
# 
# Given an input string, this function replaces all trigraph sequences. 
# The following mapping is used:
#
#     ??=    #
#     ??/    \
#     ??'    ^
#     ??(    [
#     ??)    ]
#     ??!    |
#     ??<    {
#     ??>    }
#     ??-    ~
# -----------------------------------------------------------------------------

_trigraph_pat = re.compile(r'''\?\?[=/\'\(\)\!<>\-]''')
_trigraph_rep = {
    '=':'#',
    '/':'\\',
    "'":'^',
    '(':'[',
    ')':']',
    '!':'|',
    '<':'{',
    '>':'}',
    '-':'~'
}

def trigraph(input):
    return _trigraph_pat.sub(lambda g: _trigraph_rep[g.group()[-1]],input)

def default_lexer():
    return lex.lex(optimize=in_production)

# ------------------------------------------------------------------
# Macro object
#
# This object holds information about preprocessor macros
#
#    .name      - Macro name (string)
#    .value     - Macro value (a list of tokens)
#    .arglist   - List of argument names
#    .variadic  - Boolean indicating whether or not variadic macro
#    .vararg    - Name of the variadic parameter
#
# When a macro is created, the macro replacement token sequence is
# pre-scanned and used to create patch lists that are later used
# during macro expansion
# ------------------------------------------------------------------

class Macro(object):
    def __init__(self,name,value,arglist=None,variadic=False):
        self.name = name
        self.value = value
        self.arglist = arglist
        self.variadic = variadic
        if variadic:
            self.vararg = arglist[-1]
        self.source = None
        self.lineno = None
    def __repr__(self):
        return "%s(%s)=%s" % (self.name, self.arglist, self.value)

# ------------------------------------------------------------------
# Preprocessor event hooks
#
# Override these to customise preprocessing
# ------------------------------------------------------------------

class Action(object):
    """What kind of abort processing to do in OutputDirective"""
    IgnoreAndPassThrough = 0
    """Abort processing (don't execute), but pass the directive through to output"""
    IgnoreAndRemove = 1
    """Abort processing (don't execute), and remove from output"""

class OutputDirective(Exception):
    """Raise this exception to abort processing of a preprocessor directive and
    to instead output it as is into the output"""
    def __init__(self, action):
        self.action = action

class PreprocessorHooks(object):
    """Override these in your subclass of Preprocessor to customise preprocessing"""
    def __init__(self):
        self.lastdirective = None

    def on_error(self,file,line,msg):
        """Called when the preprocessor has encountered an error, e.g. malformed input.
        
        The default simply prints to stderr and increments the return code.
        """
        print("%s:%d error: %s" % (file,line,msg), file = sys.stderr)
        self.return_code += 1
        
    def on_file_open(self,is_system_include,includepath):
        """Called to open a file for reading.
        
        This hook provides the ability to use ``chardet``, or any other mechanism,
        to inspect a file for its text encoding, and open it appropriately. Be
        aware that this function is used to probe for possible include file locations,
        so ``includepath`` may not exist. If it does not, raise the appropriate
        ``IOError`` exception.
        
        The default calls ``io.open(includepath, 'r', encoding = self.assume_encoding)``,
        examines if it starts with a BOM (if so, it removes it), and returns the file
        object opened. This raises the appropriate exception if the path was not found.
        """
        if sys.version_info.major < 3:
            assert self.assume_encoding is None
            ret = open(includepath, 'r')
        else:
            ret = open(includepath, 'r', encoding = self.assume_encoding)
        bom = ret.read(1)
        #print(repr(bom))
        if bom != '\ufeff':
            ret.seek(0)
        return ret

    def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath):
        """Called when a #include wasn't found.
        
        Raise OutputDirective to pass through or remove, else return
        a suitable path. Remember that Preprocessor.add_path() lets you add search paths.
        
        The default calls ``self.on_error()`` with a suitable error message about the
        include file not found if ``is_malformed`` is False, else a suitable error
        message about a malformed #include, and in both cases raises OutputDirective
        (pass through).
        """
        if is_malformed:
            self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Malformed #include statement: %s" % includepath)
        else:
            self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Include file '%s' not found" % includepath)
        raise OutputDirective(Action.IgnoreAndPassThrough)
        
    def on_unknown_macro_in_defined_expr(self,tok):
        """Called when an expression passed to an #if contained a defined operator
        performed on something unknown.
        
        Return True if to treat it as defined, False if to treat it as undefined,
        raise OutputDirective to pass through without execution, or return None to
        pass through the mostly expanded #if expression apart from the unknown defined.
        
        The default returns False, as per the C standard.
        """
        return False

    def on_unknown_macro_in_expr(self,ident):
        """Called when an expression passed to an #if contained an unknown identifier.
        
        Return what value the expression evaluator ought to use, or return None to
        pass through the mostly expanded #if expression.
        
        The default returns an integer 0, as per the C standard.
        """
        return 0
    
    def on_unknown_macro_function_in_expr(self,ident):
        """Called when an expression passed to an #if contained an unknown function.
        
        Return a callable which will be invoked by the expression evaluator to
        evaluate the input to the function, or return None to pass through the
        mostly expanded #if expression.
        
        The default returns a lambda which returns integer 0, as per the C standard.
        """
        return lambda x : 0
    
    def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks):
        """Called when there is one of
        
        define, include, undef, ifdef, ifndef, if, elif, else, endif
        
        Return True to execute and remove from the output, raise OutputDirective
        to pass through or remove without execution, or return None to execute
        AND pass through to the output (this only works for #define, #undef).
        
        The default returns True (execute and remove from the output).

        directive is the directive, toks is the tokens after the directive,
        ifpassthru is whether we are in passthru mode, precedingtoks is the
        tokens preceding the directive from the # token until the directive.
        """
        self.lastdirective = directive
        return True
        
    def on_directive_unknown(self,directive,toks,ifpassthru,precedingtoks):
        """Called when the preprocessor encounters a #directive it doesn't understand.
        This is actually quite an extensive list as it currently only understands:
        
        define, include, undef, ifdef, ifndef, if, elif, else, endif
        
        Return True to remove from the output, raise OutputDirective
        to pass through or remove, or return None to
        pass through into the output.
        
        The default handles #error and #warning by printing to stderr and returning True
        (remove from output). For everything else it returns None (pass through into output).

        directive is the directive, toks is the tokens after the directive,
        ifpassthru is whether we are in passthru mode, precedingtoks is the
        tokens preceding the directive from the # token until the directive.
        """
        if directive.value == 'error':
            print("%s:%d error: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
            self.return_code += 1
            return True
        elif directive.value == 'warning':
            print("%s:%d warning: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
            return True
        return None
        
    def on_potential_include_guard(self,macro):
        """Called when the preprocessor encounters an #ifndef macro or an #if !defined(macro)
        as the first non-whitespace thing in a file. Unlike the other hooks, macro is a string,
        not a token.
        """
        pass
    
    def on_comment(self,tok):
        """Called when the preprocessor encounters a comment token. You can modify the token
        in place. You must return True to let the comment pass through, else it will be removed.
        
        Returning False or None modifies the token to become whitespace, becoming a single space
        if the comment is a block comment, else a single new line if the comment is a line comment.
        """
        return None

Functions

def CPP_INTEGER(t)

(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)

Source code
def CPP_INTEGER(t):
    r'(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)'
    return t
def default_lexer()
Source code
def default_lexer():
    return lex.lex(optimize=in_production)
def t_CPP_CHAR(t)

(L)?'([^\\n]|(\(.|\n)))*?'

Source code
def t_CPP_CHAR(t):
    r'(L)?\'([^\\\n]|(\\(.|\n)))*?\''
    t.lexer.lineno += t.value.count("\n")
    return t
def t_CPP_COMMENT1(t)

(/*(.|\n)*?*/)

Source code
def t_CPP_COMMENT1(t):
    r'(/\*(.|\n)*?\*/)'
    ncr = t.value.count("\n")
    t.lexer.lineno += ncr
    return t
def t_CPP_COMMENT2(t)

(//[^\n]*)

Source code
def t_CPP_COMMENT2(t):
    r'(//[^\n]*)'
    return t
def t_CPP_INTEGER(t)

(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)

Source code
def CPP_INTEGER(t):
    r'(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)'
    return t
def t_CPP_LINECONT(t)

\[ \t]*\n

Source code
def t_CPP_LINECONT(t):
    r'\\[ \t]*\n'
    t.value = t.value[1:-1]
    t.lexer.lineno += 1
    return t
def t_CPP_STRING(t)

"([^\\n]|(\(.|\n)))*?"

Source code
def t_CPP_STRING(t):
    r'\"([^\\\n]|(\\(.|\n)))*?\"'
    t.value, subs_made = _string_literal_linecont_pat.subn('', t.value)
    t.lexer.lineno += subs_made + t.value.count("\n")
    return t
def t_CPP_WS(t)

([ \t]+|\n)

Source code
def t_CPP_WS(t):
    r'([ \t]+|\n)'
    t.lexer.lineno += t.value.count("\n")
    return t
def t_error(t)
Source code
def t_error(t):
    t.type = t.value[0]
    t.value = t.value[0]
    t.lexer.skip(1)
    return t
def trigraph(input)
Source code
def trigraph(input):
    return _trigraph_pat.sub(lambda g: _trigraph_rep[g.group()[-1]],input)

Classes

class Action

What kind of abort processing to do in OutputDirective

Source code
class Action(object):
    """What kind of abort processing to do in OutputDirective"""
    IgnoreAndPassThrough = 0
    """Abort processing (don't execute), but pass the directive through to output"""
    IgnoreAndRemove = 1
    """Abort processing (don't execute), and remove from output"""

Class variables

var IgnoreAndPassThrough

Abort processing (don't execute), but pass the directive through to output

var IgnoreAndRemove

Abort processing (don't execute), and remove from output

class Macro
Source code
class Macro(object):
    def __init__(self,name,value,arglist=None,variadic=False):
        self.name = name
        self.value = value
        self.arglist = arglist
        self.variadic = variadic
        if variadic:
            self.vararg = arglist[-1]
        self.source = None
        self.lineno = None
    def __repr__(self):
        return "%s(%s)=%s" % (self.name, self.arglist, self.value)

Methods

def __init__(self, name, value, arglist=None, variadic=False)

Initialize self. See help(type(self)) for accurate signature.

Source code
def __init__(self,name,value,arglist=None,variadic=False):
    self.name = name
    self.value = value
    self.arglist = arglist
    self.variadic = variadic
    if variadic:
        self.vararg = arglist[-1]
    self.source = None
    self.lineno = None
class OutputDirective (ancestors: builtins.Exception, builtins.BaseException)

Raise this exception to abort processing of a preprocessor directive and to instead output it as is into the output

Source code
class OutputDirective(Exception):
    """Raise this exception to abort processing of a preprocessor directive and
    to instead output it as is into the output"""
    def __init__(self, action):
        self.action = action

Methods

def __init__(self, action)

Initialize self. See help(type(self)) for accurate signature.

Source code
def __init__(self, action):
    self.action = action
class PreprocessorHooks

Override these in your subclass of Preprocessor to customise preprocessing

Source code
class PreprocessorHooks(object):
    """Override these in your subclass of Preprocessor to customise preprocessing"""
    def __init__(self):
        self.lastdirective = None

    def on_error(self,file,line,msg):
        """Called when the preprocessor has encountered an error, e.g. malformed input.
        
        The default simply prints to stderr and increments the return code.
        """
        print("%s:%d error: %s" % (file,line,msg), file = sys.stderr)
        self.return_code += 1
        
    def on_file_open(self,is_system_include,includepath):
        """Called to open a file for reading.
        
        This hook provides the ability to use ``chardet``, or any other mechanism,
        to inspect a file for its text encoding, and open it appropriately. Be
        aware that this function is used to probe for possible include file locations,
        so ``includepath`` may not exist. If it does not, raise the appropriate
        ``IOError`` exception.
        
        The default calls ``io.open(includepath, 'r', encoding = self.assume_encoding)``,
        examines if it starts with a BOM (if so, it removes it), and returns the file
        object opened. This raises the appropriate exception if the path was not found.
        """
        if sys.version_info.major < 3:
            assert self.assume_encoding is None
            ret = open(includepath, 'r')
        else:
            ret = open(includepath, 'r', encoding = self.assume_encoding)
        bom = ret.read(1)
        #print(repr(bom))
        if bom != '\ufeff':
            ret.seek(0)
        return ret

    def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath):
        """Called when a #include wasn't found.
        
        Raise OutputDirective to pass through or remove, else return
        a suitable path. Remember that Preprocessor.add_path() lets you add search paths.
        
        The default calls ``self.on_error()`` with a suitable error message about the
        include file not found if ``is_malformed`` is False, else a suitable error
        message about a malformed #include, and in both cases raises OutputDirective
        (pass through).
        """
        if is_malformed:
            self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Malformed #include statement: %s" % includepath)
        else:
            self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Include file '%s' not found" % includepath)
        raise OutputDirective(Action.IgnoreAndPassThrough)
        
    def on_unknown_macro_in_defined_expr(self,tok):
        """Called when an expression passed to an #if contained a defined operator
        performed on something unknown.
        
        Return True if to treat it as defined, False if to treat it as undefined,
        raise OutputDirective to pass through without execution, or return None to
        pass through the mostly expanded #if expression apart from the unknown defined.
        
        The default returns False, as per the C standard.
        """
        return False

    def on_unknown_macro_in_expr(self,ident):
        """Called when an expression passed to an #if contained an unknown identifier.
        
        Return what value the expression evaluator ought to use, or return None to
        pass through the mostly expanded #if expression.
        
        The default returns an integer 0, as per the C standard.
        """
        return 0
    
    def on_unknown_macro_function_in_expr(self,ident):
        """Called when an expression passed to an #if contained an unknown function.
        
        Return a callable which will be invoked by the expression evaluator to
        evaluate the input to the function, or return None to pass through the
        mostly expanded #if expression.
        
        The default returns a lambda which returns integer 0, as per the C standard.
        """
        return lambda x : 0
    
    def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks):
        """Called when there is one of
        
        define, include, undef, ifdef, ifndef, if, elif, else, endif
        
        Return True to execute and remove from the output, raise OutputDirective
        to pass through or remove without execution, or return None to execute
        AND pass through to the output (this only works for #define, #undef).
        
        The default returns True (execute and remove from the output).

        directive is the directive, toks is the tokens after the directive,
        ifpassthru is whether we are in passthru mode, precedingtoks is the
        tokens preceding the directive from the # token until the directive.
        """
        self.lastdirective = directive
        return True
        
    def on_directive_unknown(self,directive,toks,ifpassthru,precedingtoks):
        """Called when the preprocessor encounters a #directive it doesn't understand.
        This is actually quite an extensive list as it currently only understands:
        
        define, include, undef, ifdef, ifndef, if, elif, else, endif
        
        Return True to remove from the output, raise OutputDirective
        to pass through or remove, or return None to
        pass through into the output.
        
        The default handles #error and #warning by printing to stderr and returning True
        (remove from output). For everything else it returns None (pass through into output).

        directive is the directive, toks is the tokens after the directive,
        ifpassthru is whether we are in passthru mode, precedingtoks is the
        tokens preceding the directive from the # token until the directive.
        """
        if directive.value == 'error':
            print("%s:%d error: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
            self.return_code += 1
            return True
        elif directive.value == 'warning':
            print("%s:%d warning: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
            return True
        return None
        
    def on_potential_include_guard(self,macro):
        """Called when the preprocessor encounters an #ifndef macro or an #if !defined(macro)
        as the first non-whitespace thing in a file. Unlike the other hooks, macro is a string,
        not a token.
        """
        pass
    
    def on_comment(self,tok):
        """Called when the preprocessor encounters a comment token. You can modify the token
        in place. You must return True to let the comment pass through, else it will be removed.
        
        Returning False or None modifies the token to become whitespace, becoming a single space
        if the comment is a block comment, else a single new line if the comment is a line comment.
        """
        return None

Subclasses

Methods

def __init__(self)

Initialize self. See help(type(self)) for accurate signature.

Source code
def __init__(self):
    self.lastdirective = None
def on_comment(self, tok)

Called when the preprocessor encounters a comment token. You can modify the token in place. You must return True to let the comment pass through, else it will be removed.

Returning False or None modifies the token to become whitespace, becoming a single space if the comment is a block comment, else a single new line if the comment is a line comment.

Source code
def on_comment(self,tok):
    """Called when the preprocessor encounters a comment token. You can modify the token
    in place. You must return True to let the comment pass through, else it will be removed.
    
    Returning False or None modifies the token to become whitespace, becoming a single space
    if the comment is a block comment, else a single new line if the comment is a line comment.
    """
    return None
def on_directive_handle(self, directive, toks, ifpassthru, precedingtoks)

Called when there is one of

define, include, undef, ifdef, ifndef, if, elif, else, endif

Return True to execute and remove from the output, raise OutputDirective to pass through or remove without execution, or return None to execute AND pass through to the output (this only works for #define, #undef).

The default returns True (execute and remove from the output).

directive is the directive, toks is the tokens after the directive, ifpassthru is whether we are in passthru mode, precedingtoks is the tokens preceding the directive from the # token until the directive.

Source code
def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks):
    """Called when there is one of
    
    define, include, undef, ifdef, ifndef, if, elif, else, endif
    
    Return True to execute and remove from the output, raise OutputDirective
    to pass through or remove without execution, or return None to execute
    AND pass through to the output (this only works for #define, #undef).
    
    The default returns True (execute and remove from the output).

    directive is the directive, toks is the tokens after the directive,
    ifpassthru is whether we are in passthru mode, precedingtoks is the
    tokens preceding the directive from the # token until the directive.
    """
    self.lastdirective = directive
    return True
def on_directive_unknown(self, directive, toks, ifpassthru, precedingtoks)

Called when the preprocessor encounters a #directive it doesn't understand. This is actually quite an extensive list as it currently only understands:

define, include, undef, ifdef, ifndef, if, elif, else, endif

Return True to remove from the output, raise OutputDirective to pass through or remove, or return None to pass through into the output.

The default handles #error and #warning by printing to stderr and returning True (remove from output). For everything else it returns None (pass through into output).

directive is the directive, toks is the tokens after the directive, ifpassthru is whether we are in passthru mode, precedingtoks is the tokens preceding the directive from the # token until the directive.

Source code
def on_directive_unknown(self,directive,toks,ifpassthru,precedingtoks):
    """Called when the preprocessor encounters a #directive it doesn't understand.
    This is actually quite an extensive list as it currently only understands:
    
    define, include, undef, ifdef, ifndef, if, elif, else, endif
    
    Return True to remove from the output, raise OutputDirective
    to pass through or remove, or return None to
    pass through into the output.
    
    The default handles #error and #warning by printing to stderr and returning True
    (remove from output). For everything else it returns None (pass through into output).

    directive is the directive, toks is the tokens after the directive,
    ifpassthru is whether we are in passthru mode, precedingtoks is the
    tokens preceding the directive from the # token until the directive.
    """
    if directive.value == 'error':
        print("%s:%d error: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
        self.return_code += 1
        return True
    elif directive.value == 'warning':
        print("%s:%d warning: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
        return True
    return None
def on_error(self, file, line, msg)

Called when the preprocessor has encountered an error, e.g. malformed input.

The default simply prints to stderr and increments the return code.

Source code
def on_error(self,file,line,msg):
    """Called when the preprocessor has encountered an error, e.g. malformed input.
    
    The default simply prints to stderr and increments the return code.
    """
    print("%s:%d error: %s" % (file,line,msg), file = sys.stderr)
    self.return_code += 1
def on_file_open(self, is_system_include, includepath)

Called to open a file for reading.

This hook provides the ability to use chardet, or any other mechanism, to inspect a file for its text encoding, and open it appropriately. Be aware that this function is used to probe for possible include file locations, so includepath may not exist. If it does not, raise the appropriate IOError exception.

The default calls io.open(includepath, 'r', encoding = self.assume_encoding), examines if it starts with a BOM (if so, it removes it), and returns the file object opened. This raises the appropriate exception if the path was not found.

Source code
def on_file_open(self,is_system_include,includepath):
    """Called to open a file for reading.
    
    This hook provides the ability to use ``chardet``, or any other mechanism,
    to inspect a file for its text encoding, and open it appropriately. Be
    aware that this function is used to probe for possible include file locations,
    so ``includepath`` may not exist. If it does not, raise the appropriate
    ``IOError`` exception.
    
    The default calls ``io.open(includepath, 'r', encoding = self.assume_encoding)``,
    examines if it starts with a BOM (if so, it removes it), and returns the file
    object opened. This raises the appropriate exception if the path was not found.
    """
    if sys.version_info.major < 3:
        assert self.assume_encoding is None
        ret = open(includepath, 'r')
    else:
        ret = open(includepath, 'r', encoding = self.assume_encoding)
    bom = ret.read(1)
    #print(repr(bom))
    if bom != '\ufeff':
        ret.seek(0)
    return ret
def on_include_not_found(self, is_malformed, is_system_include, curdir, includepath)

Called when a #include wasn't found.

Raise OutputDirective to pass through or remove, else return a suitable path. Remember that Preprocessor.add_path() lets you add search paths.

The default calls self.on_error() with a suitable error message about the include file not found if is_malformed is False, else a suitable error message about a malformed #include, and in both cases raises OutputDirective (pass through).

Source code
def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath):
    """Called when a #include wasn't found.
    
    Raise OutputDirective to pass through or remove, else return
    a suitable path. Remember that Preprocessor.add_path() lets you add search paths.
    
    The default calls ``self.on_error()`` with a suitable error message about the
    include file not found if ``is_malformed`` is False, else a suitable error
    message about a malformed #include, and in both cases raises OutputDirective
    (pass through).
    """
    if is_malformed:
        self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Malformed #include statement: %s" % includepath)
    else:
        self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Include file '%s' not found" % includepath)
    raise OutputDirective(Action.IgnoreAndPassThrough)
def on_potential_include_guard(self, macro)

Called when the preprocessor encounters an #ifndef macro or an #if !defined(macro) as the first non-whitespace thing in a file. Unlike the other hooks, macro is a string, not a token.

Source code
def on_potential_include_guard(self,macro):
    """Called when the preprocessor encounters an #ifndef macro or an #if !defined(macro)
    as the first non-whitespace thing in a file. Unlike the other hooks, macro is a string,
    not a token.
    """
    pass
def on_unknown_macro_function_in_expr(self, ident)

Called when an expression passed to an #if contained an unknown function.

Return a callable which will be invoked by the expression evaluator to evaluate the input to the function, or return None to pass through the mostly expanded #if expression.

The default returns a lambda which returns integer 0, as per the C standard.

Source code
def on_unknown_macro_function_in_expr(self,ident):
    """Called when an expression passed to an #if contained an unknown function.
    
    Return a callable which will be invoked by the expression evaluator to
    evaluate the input to the function, or return None to pass through the
    mostly expanded #if expression.
    
    The default returns a lambda which returns integer 0, as per the C standard.
    """
    return lambda x : 0
def on_unknown_macro_in_defined_expr(self, tok)

Called when an expression passed to an #if contained a defined operator performed on something unknown.

Return True if to treat it as defined, False if to treat it as undefined, raise OutputDirective to pass through without execution, or return None to pass through the mostly expanded #if expression apart from the unknown defined.

The default returns False, as per the C standard.

Source code
def on_unknown_macro_in_defined_expr(self,tok):
    """Called when an expression passed to an #if contained a defined operator
    performed on something unknown.
    
    Return True if to treat it as defined, False if to treat it as undefined,
    raise OutputDirective to pass through without execution, or return None to
    pass through the mostly expanded #if expression apart from the unknown defined.
    
    The default returns False, as per the C standard.
    """
    return False
def on_unknown_macro_in_expr(self, ident)

Called when an expression passed to an #if contained an unknown identifier.

Return what value the expression evaluator ought to use, or return None to pass through the mostly expanded #if expression.

The default returns an integer 0, as per the C standard.

Source code
def on_unknown_macro_in_expr(self,ident):
    """Called when an expression passed to an #if contained an unknown identifier.
    
    Return what value the expression evaluator ought to use, or return None to
    pass through the mostly expanded #if expression.
    
    The default returns an integer 0, as per the C standard.
    """
    return 0
================================================ FILE: doc/parsetab.html ================================================ pcpp.parsetab API documentation

pcpp.parsetab module

Source code
# parsetab.py
# This file is automatically generated. Do not edit.
# pylint: disable=W,C,R
_tabversion = '3.10'

_lr_method = 'LALR'

_lr_signature = 'leftCPP_COMMAleftCPP_QUESTIONCPP_COLONleftCPP_LOGICALORleftCPP_LOGICALANDleftCPP_BARleftCPP_HATleftCPP_AMPERSANDleftCPP_EQUALITYCPP_INEQUALITYleftCPP_LESSCPP_LESSEQUALCPP_GREATERCPP_GREATEREQUALleftCPP_LSHIFTCPP_RSHIFTleftCPP_PLUSCPP_MINUSleftCPP_STARCPP_FSLASHCPP_PERCENTrightUPLUSUMINUSCPP_EXCLAMATIONCPP_TILDECPP_AMPERSAND CPP_BAR CPP_CHAR CPP_COLON CPP_COMMA CPP_EQUALITY CPP_EXCLAMATION CPP_FSLASH CPP_GREATER CPP_GREATEREQUAL CPP_HAT CPP_ID CPP_INEQUALITY CPP_INTEGER CPP_LESS CPP_LESSEQUAL CPP_LOGICALAND CPP_LOGICALOR CPP_LPAREN CPP_LSHIFT CPP_MINUS CPP_PERCENT CPP_PLUS CPP_QUESTION CPP_RPAREN CPP_RSHIFT CPP_STAR CPP_STRING CPP_TILDEexpression : CPP_INTEGERexpression : CPP_CHAR\n    expression : CPP_STRING\n              | CPP_LESS expression CPP_GREATER\n    expression : CPP_LPAREN expression CPP_RPARENexpression : CPP_PLUS expression %prec UPLUSexpression : CPP_MINUS expression %prec UMINUS\n    expression : CPP_EXCLAMATION expression\n              | CPP_TILDE expression\n    \n    expression : expression CPP_STAR expression\n              | expression CPP_FSLASH expression\n              | expression CPP_PERCENT expression\n              | expression CPP_PLUS expression\n              | expression CPP_MINUS expression\n              | expression CPP_LSHIFT expression\n              | expression CPP_RSHIFT expression\n              | expression CPP_LESS expression\n              | expression CPP_LESSEQUAL expression\n              | expression CPP_GREATER expression\n              | expression CPP_GREATEREQUAL expression\n              | expression CPP_EQUALITY expression\n              | expression CPP_INEQUALITY expression\n              | expression CPP_AMPERSAND expression\n              | expression CPP_HAT expression\n              | expression CPP_BAR expression\n              | expression CPP_LOGICALAND expression\n              | expression CPP_LOGICALOR expression\n              | expression CPP_COMMA expression\n    expression : expression CPP_QUESTION expression CPP_COLON expressionexpression : CPP_ID CPP_LPAREN expression CPP_RPARENexpression : CPP_ID'
    
_lr_action_items = {'CPP_ID':([0,3,4,6,8,9,11,12,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,41,63,],[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,]),'CPP_LSHIFT':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,18,-8,-6,18,-7,18,-9,18,-5,-4,-15,18,18,18,18,-12,18,18,18,-13,18,-11,18,18,-14,-10,18,18,18,-16,-30,18,]),'CPP_STRING':([0,3,4,6,8,9,11,12,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,41,63,],[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,]),'CPP_LESSEQUAL':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,19,-8,-6,19,-7,19,-9,19,-5,-4,-15,-18,-20,19,19,-12,19,19,19,-13,19,-11,19,19,-14,-10,19,-19,-17,-16,-30,19,]),'CPP_GREATEREQUAL':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,20,-8,-6,20,-7,20,-9,20,-5,-4,-15,-18,-20,20,20,-12,20,20,20,-13,20,-11,20,20,-14,-10,20,-19,-17,-16,-30,20,]),'CPP_LOGICALOR':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,21,-8,-6,21,-7,21,-9,21,-5,-4,-15,-18,-20,-27,-26,-12,-23,21,-25,-13,-22,-11,-21,-24,-14,-10,21,-19,-17,-16,-30,21,]),'CPP_LOGICALAND':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,22,-8,-6,22,-7,22,-9,22,-5,-4,-15,-18,-20,22,-26,-12,-23,22,-25,-13,-22,-11,-21,-24,-14,-10,22,-19,-17,-16,-30,22,]),'CPP_EXCLAMATION':([0,3,4,6,8,9,11,12,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,41,63,],[3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,]),'CPP_PERCENT':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,23,-8,-6,23,-7,23,-9,23,-5,-4,23,23,23,23,23,-12,23,23,23,23,23,-11,23,23,23,-10,23,23,23,23,-30,23,]),'CPP_AMPERSAND':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,24,-8,-6,24,-7,24,-9,24,-5,-4,-15,-18,-20,24,24,-12,-23,24,24,-13,-22,-11,-21,24,-14,-10,24,-19,-17,-16,-30,24,]),'CPP_QUESTION':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,25,-8,-6,25,-7,25,-9,25,-5,-4,-15,-18,-20,-27,-26,-12,-23,25,-25,-13,-22,-11,-21,-24,-14,-10,25,-19,-17,-16,-30,-29,]),'CPP_BAR':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,26,-8,-6,26,-7,26,-9,26,-5,-4,-15,-18,-20,26,26,-12,-23,26,-25,-13,-22,-11,-21,-24,-14,-10,26,-19,-17,-16,-30,26,]),'$end':([1,2,5,7,10,13,14,16,38,40,41,42,43,44,45,46,47,48,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,0,-8,-6,-7,-9,-5,-4,-15,-18,-20,-27,-26,-12,-23,-25,-13,-22,-11,-21,-24,-14,-10,-28,-19,-17,-16,-30,-29,]),'CPP_PLUS':([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,],[4,-31,-3,4,4,-2,4,-1,4,4,27,4,4,-8,-6,27,-7,27,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,-9,27,-5,4,27,27,27,27,27,-12,27,27,27,-13,27,-11,27,27,-14,-10,27,27,27,27,-30,4,27,]),'CPP_INEQUALITY':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,28,-8,-6,28,-7,28,-9,28,-5,-4,-15,-18,-20,28,28,-12,28,28,28,-13,-22,-11,-21,28,-14,-10,28,-19,-17,-16,-30,28,]),'CPP_CHAR':([0,3,4,6,8,9,11,12,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,41,63,],[5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,]),'CPP_LPAREN':([0,1,3,4,6,8,9,11,12,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,41,63,],[6,12,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,]),'CPP_FSLASH':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,29,-8,-6,29,-7,29,-9,29,-5,-4,29,29,29,29,29,-12,29,29,29,29,29,-11,29,29,29,-10,29,29,29,29,-30,29,]),'CPP_INTEGER':([0,3,4,6,8,9,11,12,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,41,63,],[7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,]),'CPP_HAT':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,31,-8,-6,31,-7,31,-9,31,-5,-4,-15,-18,-20,31,31,-12,-23,31,31,-13,-22,-11,-21,-24,-14,-10,31,-19,-17,-16,-30,31,]),'CPP_EQUALITY':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,30,-8,-6,30,-7,30,-9,30,-5,-4,-15,-18,-20,30,30,-12,30,30,30,-13,-22,-11,-21,30,-14,-10,30,-19,-17,-16,-30,30,]),'CPP_MINUS':([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,],[8,-31,-3,8,8,-2,8,-1,8,8,32,8,8,-8,-6,32,-7,32,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,-9,32,-5,8,32,32,32,32,32,-12,32,32,32,-13,32,-11,32,32,-14,-10,32,32,32,32,-30,8,32,]),'CPP_STAR':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,33,-8,-6,33,-7,33,-9,33,-5,-4,33,33,33,33,33,-12,33,33,33,33,33,-11,33,33,33,-10,33,33,33,33,-30,33,]),'CPP_COMMA':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,34,-8,-6,34,-7,34,-9,34,-5,-4,-15,-18,-20,-27,-26,-12,-23,34,-25,-13,-22,-11,-21,-24,-14,-10,-28,-19,-17,-16,-30,-29,]),'CPP_GREATER':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,35,-8,-6,35,-7,41,-9,35,-5,-4,-15,-18,-20,35,35,-12,35,35,35,-13,35,-11,35,35,-14,-10,35,-19,-17,-16,-30,35,]),'CPP_RSHIFT':([1,2,5,7,10,13,14,15,16,17,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,37,-8,-6,37,-7,37,-9,37,-5,-4,-15,37,37,37,37,-12,37,37,37,-13,37,-11,37,37,-14,-10,37,37,37,-16,-30,37,]),'CPP_LESS':([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,],[9,-31,-3,9,9,-2,9,-1,9,9,36,9,9,-8,-6,36,-7,36,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,-9,36,-5,-4,-15,-18,-20,36,36,-12,36,36,36,-13,36,-11,36,36,-14,-10,36,-19,-17,-16,-30,9,36,]),'CPP_COLON':([1,2,5,7,13,14,16,38,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,-8,-6,-7,-9,-5,-4,-15,-18,-20,-27,-26,-12,-23,63,-25,-13,-22,-11,-21,-24,-14,-10,-28,-19,-17,-16,-30,-29,]),'CPP_TILDE':([0,3,4,6,8,9,11,12,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,41,63,],[11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,]),'CPP_RPAREN':([1,2,5,7,13,14,15,16,38,39,40,41,42,43,44,45,46,47,48,50,51,52,53,54,55,56,57,58,59,60,61,62,64,],[-31,-3,-2,-1,-8,-6,40,-7,-9,62,-5,-4,-15,-18,-20,-27,-26,-12,-23,-25,-13,-22,-11,-21,-24,-14,-10,-28,-19,-17,-16,-30,-29,]),}

_lr_action = {}
for _k, _v in _lr_action_items.items():
   for _x,_y in zip(_v[0],_v[1]):
      if not _x in _lr_action:  _lr_action[_x] = {}
      _lr_action[_x][_k] = _y
del _lr_action_items

_lr_goto_items = {'expression':([0,3,4,6,8,9,11,12,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,41,63,],[10,13,14,15,16,17,38,39,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,59,64,]),}

_lr_goto = {}
for _k, _v in _lr_goto_items.items():
   for _x, _y in zip(_v[0], _v[1]):
       if not _x in _lr_goto: _lr_goto[_x] = {}
       _lr_goto[_x][_k] = _y
del _lr_goto_items
_lr_productions = [
  ("S' -> expression","S'",1,None,None,None),
  ('expression -> CPP_INTEGER','expression',1,'p_expression_number','evaluator.py',396),
  ('expression -> CPP_CHAR','expression',1,'p_expression_character','evaluator.py',400),
  ('expression -> CPP_STRING','expression',1,'p_expression_string','evaluator.py',405),
  ('expression -> CPP_LESS expression CPP_GREATER','expression',3,'p_expression_string','evaluator.py',406),
  ('expression -> CPP_LPAREN expression CPP_RPAREN','expression',3,'p_expression_group','evaluator.py',411),
  ('expression -> CPP_PLUS expression','expression',2,'p_expression_uplus','evaluator.py',415),
  ('expression -> CPP_MINUS expression','expression',2,'p_expression_uminus','evaluator.py',419),
  ('expression -> CPP_EXCLAMATION expression','expression',2,'p_expression_unop','evaluator.py',424),
  ('expression -> CPP_TILDE expression','expression',2,'p_expression_unop','evaluator.py',425),
  ('expression -> expression CPP_STAR expression','expression',3,'p_expression_binop','evaluator.py',437),
  ('expression -> expression CPP_FSLASH expression','expression',3,'p_expression_binop','evaluator.py',438),
  ('expression -> expression CPP_PERCENT expression','expression',3,'p_expression_binop','evaluator.py',439),
  ('expression -> expression CPP_PLUS expression','expression',3,'p_expression_binop','evaluator.py',440),
  ('expression -> expression CPP_MINUS expression','expression',3,'p_expression_binop','evaluator.py',441),
  ('expression -> expression CPP_LSHIFT expression','expression',3,'p_expression_binop','evaluator.py',442),
  ('expression -> expression CPP_RSHIFT expression','expression',3,'p_expression_binop','evaluator.py',443),
  ('expression -> expression CPP_LESS expression','expression',3,'p_expression_binop','evaluator.py',444),
  ('expression -> expression CPP_LESSEQUAL expression','expression',3,'p_expression_binop','evaluator.py',445),
  ('expression -> expression CPP_GREATER expression','expression',3,'p_expression_binop','evaluator.py',446),
  ('expression -> expression CPP_GREATEREQUAL expression','expression',3,'p_expression_binop','evaluator.py',447),
  ('expression -> expression CPP_EQUALITY expression','expression',3,'p_expression_binop','evaluator.py',448),
  ('expression -> expression CPP_INEQUALITY expression','expression',3,'p_expression_binop','evaluator.py',449),
  ('expression -> expression CPP_AMPERSAND expression','expression',3,'p_expression_binop','evaluator.py',450),
  ('expression -> expression CPP_HAT expression','expression',3,'p_expression_binop','evaluator.py',451),
  ('expression -> expression CPP_BAR expression','expression',3,'p_expression_binop','evaluator.py',452),
  ('expression -> expression CPP_LOGICALAND expression','expression',3,'p_expression_binop','evaluator.py',453),
  ('expression -> expression CPP_LOGICALOR expression','expression',3,'p_expression_binop','evaluator.py',454),
  ('expression -> expression CPP_COMMA expression','expression',3,'p_expression_binop','evaluator.py',455),
  ('expression -> expression CPP_QUESTION expression CPP_COLON expression','expression',5,'p_expression_conditional','evaluator.py',501),
  ('expression -> CPP_ID CPP_LPAREN expression CPP_RPAREN','expression',4,'p_expression_function_call','evaluator.py',513),
  ('expression -> CPP_ID','expression',1,'p_expression_identifier','evaluator.py',520),
]
================================================ FILE: doc/pcmd.html ================================================ pcpp.pcmd API documentation

pcpp.pcmd module

Source code
#!/usr/bin/python
# Python C99 conforming preprocessor command line
# (C) 2017-2026 Niall Douglas http://www.nedproductions.biz/
# Started: March 2017

from __future__ import generators, print_function, absolute_import, division

import sys, argparse, traceback, os, copy, io, re
if __name__ == '__main__' and __package__ is None:
    sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) )
from pcpp.preprocessor import Preprocessor, OutputDirective, Action

version='1.30'

__all__ = []

class FileAction(argparse.Action):
    def __init__(self, option_strings, dest, **kwargs):
        super(FileAction, self).__init__(option_strings, dest, **kwargs)
        
    def __call__(self, parser, namespace, values, option_string=None):
        if getattr(namespace, self.dest)[0] == sys.stdin:
            items = []
        else:
            items = copy.copy(getattr(namespace, self.dest))
        items += [argparse.FileType('rt')(value) for value in values]
        setattr(namespace, self.dest, items)

class CmdPreprocessor(Preprocessor):
    def __init__(self, argv):
        if len(argv) < 2:
            argv = [argv[0], '--help']
        argp = argparse.ArgumentParser(prog='pcpp',
            description=
    '''A pure universal Python C (pre-)preprocessor implementation very useful for
    pre-preprocessing header only C++ libraries into single file includes and
    other such build or packaging stage malarky.''',
            epilog=
    '''Note that so pcpp can stand in for other preprocessor tooling, it
    ignores any arguments it does not understand.''')
        argp.add_argument('inputs', metavar = 'input', default = [sys.stdin], nargs = '*', action = FileAction, help = 'Files to preprocess (use \'-\' for stdin)')
        argp.add_argument('-o', dest = 'output', metavar = 'path', type = argparse.FileType('wt'), default=sys.stdout, nargs = '?', help = 'Output to a file instead of stdout')
        argp.add_argument('-D', dest = 'defines', metavar = 'macro[=val]', nargs = 1, action = 'append', help = 'Predefine name as a macro [with value]')
        argp.add_argument('-U', dest = 'undefines', metavar = 'macro', nargs = 1, action = 'append', help = 'Pre-undefine name as a macro')
        argp.add_argument('-N', dest = 'nevers', metavar = 'macro', nargs = 1, action = 'append', help = 'Never define name as a macro, even if defined during the preprocessing.')
        argp.add_argument('-I', dest = 'includes', metavar = 'path', nargs = 1, action = 'append', help = "Path to search for unfound #include's")
        #argp.add_argument('--passthru', dest = 'passthru', action = 'store_true', help = 'Pass through everything unexecuted except for #include and include guards (which need to be the first thing in an include file')
        argp.add_argument('--passthru-defines', dest = 'passthru_defines', action = 'store_true', help = 'Pass through but still execute #defines and #undefs if not always removed by preprocessor logic')
        argp.add_argument('--passthru-unfound-includes', dest = 'passthru_unfound_includes', action = 'store_true', help = 'Pass through #includes not found without execution')
        argp.add_argument('--passthru-unknown-exprs', dest = 'passthru_undefined_exprs', action = 'store_true', help = 'Unknown macros in expressions cause preprocessor logic to be passed through instead of executed by treating unknown macros as 0L')
        argp.add_argument('--passthru-comments', dest = 'passthru_comments', action = 'store_true', help = 'Pass through comments unmodified')
        argp.add_argument('--passthru-magic-macros', dest = 'passthru_magic_macros', action = 'store_true', help = 'Pass through double underscore magic macros unmodified')
        argp.add_argument('--passthru-includes', dest = 'passthru_includes', metavar = '<regex>', default = None, nargs = 1, help = "Regular expression for which #includes to not expand. #includes, if found, are always executed")
        argp.add_argument('--disable-auto-pragma-once', dest = 'auto_pragma_once_disabled', action = 'store_true', default = False, help = 'Disable the heuristics which auto apply #pragma once to #include files wholly wrapped in an obvious include guard macro')
        argp.add_argument('--line-directive', dest = 'line_directive', metavar = 'form', default = '#line', nargs = '?', help = "Form of line directive to use, defaults to #line, specify nothing to disable output of line directives")
        argp.add_argument('--debug', dest = 'debug', action = 'store_true', help = 'Generate a pcpp_debug.log file logging execution')
        argp.add_argument('--time', dest = 'time', action = 'store_true', help = 'Print the time it took to #include each file')
        argp.add_argument('--filetimes', dest = 'filetimes', metavar = 'path', type = argparse.FileType('wt'), default=None, nargs = '?', help = 'Write CSV file with time spent inside each included file, inclusive and exclusive')
        argp.add_argument('--compress', dest = 'compress', action = 'store_true', help = 'Make output as small as possible')
        argp.add_argument('--assume-input-encoding', dest = 'assume_input_encoding', metavar = '<encoding>', default = None, nargs = 1, help = 'The text encoding to assume inputs are in')
        argp.add_argument('--output-encoding', dest = 'output_encoding', metavar = '<encoding>', default = None, nargs = 1, help = 'The text encoding to use when writing files')
        argp.add_argument('--write-bom', dest = 'write_bom', action = 'store_true', help = 'Prefix any output with a Unicode BOM')
        argp.add_argument('--version', action='version', version='pcpp ' + version)
        args = argp.parse_known_args(argv[1:])
        #print(args)
        for arg in args[1]:
            print("NOTE: Argument %s not known, ignoring!" % arg, file = sys.stderr)

        self.args = args[0]
        super(CmdPreprocessor, self).__init__()
        
        # Override Preprocessor instance variables
        self.define("__PCPP_VERSION__ " + version)
        self.define("__PCPP_ALWAYS_FALSE__ 0")
        self.define("__PCPP_ALWAYS_TRUE__ 1")
        if self.args.debug:
            self.debugout = open("pcpp_debug.log", "wt")
        self.auto_pragma_once_enabled = not self.args.auto_pragma_once_disabled
        self.line_directive = self.args.line_directive
        if self.line_directive is not None and self.line_directive.lower() in ('nothing', 'none', ''):
            self.line_directive = None
        if self.args.passthru_includes is not None:
            self.passthru_includes = re.compile(self.args.passthru_includes[0])
        self.compress = 2 if self.args.compress else 0
        if self.args.passthru_magic_macros:
            self.undef('__DATE__')
            self.undef('__TIME__')
            self.expand_linemacro = False
            self.expand_filemacro = False
            self.expand_countermacro = False
        if self.args.assume_input_encoding is not None:
            self.args.assume_input_encoding = self.args.assume_input_encoding[0]
            self.assume_encoding = self.args.assume_input_encoding
            if len(self.args.inputs) == 1:
                # Reopen our input files with the appropriate encoding
                _ = self.on_file_open(False, self.args.inputs[0].name)
                self.args.inputs[0].close()
                self.args.inputs[0] = _
            if self.args.output_encoding is None:
                self.args.output_encoding = self.args.assume_input_encoding
        if self.args.output_encoding is not None:
            self.args.output_encoding = self.args.output_encoding[0]
            # Reopen our output file with the appropriate encoding
            _ = io.open(self.args.output.name, 'w', encoding = self.args.output_encoding)
            self.args.output.close()
            self.args.output = _
            if self.args.write_bom:
                self.args.output.write('\ufeff')
        
        # My own instance variables
        self.bypass_ifpassthru = False
        self.potential_include_guard = None

        if self.args.defines:
            self.args.defines = [x[0] for x in self.args.defines]
            for d in self.args.defines:
                if '=' not in d:
                    d += '=1'
                d = d.replace('=', ' ', 1)
                self.define(d)
        if self.args.undefines:
            self.args.undefines = [x[0] for x in self.args.undefines]
            for d in self.args.undefines:
                self.undef(d)
        if self.args.nevers:
            self.args.nevers = [x[0] for x in self.args.nevers]
        if self.args.includes:
            self.args.includes = [x[0] for x in self.args.includes]
            for d in self.args.includes:
                self.add_path(d)

        try:
            if len(self.args.inputs) == 1:
                self.parse(self.args.inputs[0])
            else:
                input = ''
                for i in self.args.inputs:
                    input += '#include "' + i.name + '"\n'
                self.parse(input)
            self.write(self.args.output)
        except:
            print(traceback.print_exc(10), file = sys.stderr)
            print("\nINTERNAL PREPROCESSOR ERROR AT AROUND %s:%d, FATALLY EXITING NOW\n"
                % (self.lastdirective.source, self.lastdirective.lineno), file = sys.stderr)
            sys.exit(-99)
        finally:
            for i in self.args.inputs:
                i.close()
            if self.args.output != sys.stdout:
                self.args.output.close()
        
        if self.args.time:
            print("\nTime report:")
            print("============")
            for n in range(0, len(self.include_times)):
                if n == 0:
                    print("top level: %f seconds" % self.include_times[n].elapsed)
                elif self.include_times[n].depth == 1:
                    print("\n %s: %f seconds (%f%%)" % (self.include_times[n].included_path, self.include_times[n].elapsed, 100 * self.include_times[n].elapsed / self.include_times[0].elapsed))
                else:
                    print("%s%s: %f seconds" % (' ' * self.include_times[n].depth, self.include_times[n].included_path, self.include_times[n].elapsed))
            print("\nPragma once files (including heuristically applied):")
            print("====================================================")
            for i in self.include_once:
                print(" ", i)
            print()
        if self.args.filetimes:
            print('"Total seconds","Self seconds","File size","File path"', file = self.args.filetimes)
            filetimes = {}
            currentfiles = []
            for n in range(0, len(self.include_times)):
                while self.include_times[n].depth < len(currentfiles):
                    currentfiles.pop()
                if self.include_times[n].depth > len(currentfiles) - 1:
                    currentfiles.append(self.include_times[n].included_abspath)
                #print()
                #for path in currentfiles:
                #    print("currentfiles =", path)
                path = currentfiles[-1]
                if path in filetimes:
                    filetimes[path][0] += self.include_times[n].elapsed
                    filetimes[path][1] += self.include_times[n].elapsed
                else:
                    filetimes[path] = [self.include_times[n].elapsed, self.include_times[n].elapsed]
                if self.include_times[n].elapsed > 0 and len(currentfiles) > 1:
                    #print("Removing child %f from parent %s = %f" % (self.include_times[n].elapsed, currentfiles[-2], filetimes[currentfiles[-2]]))
                    filetimes[currentfiles[-2]][1] -= self.include_times[n].elapsed
            filetimes = [(v[0],v[1],k) for k,v in filetimes.items()]
            filetimes.sort(reverse=True)
            for t,s,p in filetimes:
                print(('%f,%f,%d,"%s"' % (t, s, os.stat(p).st_size, p)), file = self.args.filetimes)
    def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath):
        if self.args.passthru_unfound_includes:
            raise OutputDirective(Action.IgnoreAndPassThrough)
        return super(CmdPreprocessor, self).on_include_not_found(is_malformed,is_system_include,curdir,includepath)

    def on_unknown_macro_in_defined_expr(self,tok):
        if self.args.undefines:
            if tok.value in self.args.undefines:
                return False
        if self.args.passthru_undefined_exprs:
            return None  # Pass through as expanded as possible
        return super(CmdPreprocessor, self).on_unknown_macro_in_defined_expr(tok)
        
    def on_unknown_macro_in_expr(self,ident):
        if self.args.undefines:
            if ident in self.args.undefines:
                return super(CmdPreprocessor, self).on_unknown_macro_in_expr(ident)
        if self.args.passthru_undefined_exprs:
            return None  # Pass through as expanded as possible
        return super(CmdPreprocessor, self).on_unknown_macro_in_expr(ident)
        
    def on_unknown_macro_function_in_expr(self,ident):
        if self.args.undefines:
            if ident in self.args.undefines:
                return super(CmdPreprocessor, self).on_unknown_macro_function_in_expr(ident)
        if self.args.passthru_undefined_exprs:
            return None  # Pass through as expanded as possible
        return super(CmdPreprocessor, self).on_unknown_macro_function_in_expr(ident)
        
    def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks):
        if ifpassthru:
            if directive.value == 'if' or directive.value == 'elif' or directive == 'else' or directive.value == 'endif':
                self.bypass_ifpassthru = len([tok for tok in toks if tok.value == '__PCPP_ALWAYS_FALSE__' or tok.value == '__PCPP_ALWAYS_TRUE__']) > 0
            if not self.bypass_ifpassthru and (directive.value == 'define' or directive.value == 'undef'):
                if toks[0].value != self.potential_include_guard:
                    raise OutputDirective(Action.IgnoreAndPassThrough)  # Don't execute anything with effects when inside an #if expr with undefined macro
        if (directive.value == 'define' or directive.value == 'undef') and self.args.nevers:
            if toks[0].value in self.args.nevers:
                raise OutputDirective(Action.IgnoreAndPassThrough)
        if self.args.passthru_defines:
            super(CmdPreprocessor, self).on_directive_handle(directive,toks,ifpassthru,precedingtoks)
            return None  # Pass through where possible
        return super(CmdPreprocessor, self).on_directive_handle(directive,toks,ifpassthru,precedingtoks)

    def on_directive_unknown(self,directive,toks,ifpassthru,precedingtoks):
        if ifpassthru:
            return None  # Pass through
        return super(CmdPreprocessor, self).on_directive_unknown(directive,toks,ifpassthru,precedingtoks)

    def on_potential_include_guard(self,macro):
        self.potential_include_guard = macro
        return super(CmdPreprocessor, self).on_potential_include_guard(macro)

    def on_comment(self,tok):
        if self.args.passthru_comments:
            return True  # Pass through
        return super(CmdPreprocessor, self).on_comment(tok)

def main():
    p = CmdPreprocessor(sys.argv)
    sys.exit(p.return_code)
        
if __name__ == "__main__":
    p = CmdPreprocessor(sys.argv)
    sys.exit(p.return_code)
================================================ FILE: doc/preprocessor.html ================================================ pcpp.preprocessor API documentation

pcpp.preprocessor module

Source code
#!/usr/bin/python
# Python C99 conforming preprocessor useful for generating single include files
# (C) 2017-2026 Niall Douglas http://www.nedproductions.biz/
# and (C) 2007-2017 David Beazley http://www.dabeaz.com/
# Started: Feb 2017
#
# This C preprocessor was originally written by David Beazley and the
# original can be found at https://github.com/dabeaz/ply/blob/master/ply/cpp.py
# This edition substantially improves on standards conforming output,
# getting quite close to what clang or GCC outputs.

from __future__ import generators, print_function, absolute_import, division

import sys, os, re, codecs, time, copy, traceback
if __name__ == '__main__' and __package__ is None:
    sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) )
from pcpp.parser import STRING_TYPES, default_lexer, trigraph, Macro, Action, OutputDirective, PreprocessorHooks
from pcpp.evaluator import Evaluator

# Some Python 3 compatibility shims
if sys.version_info.major < 3:
    FILE_TYPES = file
    clock = time.clock
else:
    xrange = range
    import io
    FILE_TYPES = io.IOBase
    clock = time.process_time

__all__ = ['Preprocessor', 'PreprocessorHooks', 'OutputDirective', 'Action', 'Evaluator']

# ------------------------------------------------------------------
# File inclusion timings
#
# Useful for figuring out how long a sequence of preprocessor inclusions actually is
# ------------------------------------------------------------------

class FileInclusionTime(object):
    """The seconds taken to #include another file"""
    def __init__(self,including_path,included_path,included_abspath,depth):
        self.including_path = including_path
        self.included_path = included_path
        self.included_abspath = included_abspath
        self.depth = depth
        self.elapsed = 0.0

# ------------------------------------------------------------------
# Preprocessor object
#
# Object representing a preprocessor.  Contains macro definitions,
# include directories, and other information
# ------------------------------------------------------------------

class Preprocessor(PreprocessorHooks):    
    def __init__(self,lexer=None):
        super(Preprocessor, self).__init__()
        if lexer is None:
            lexer = default_lexer()
        self.lexer = lexer
        self.evaluator = Evaluator(self.lexer)
        self.macros = { }
        self.path = []           # list of -I formal search paths for includes
        self.temp_path = []      # list of temporary search paths for includes
        self.rewrite_paths = [(re.escape(os.path.abspath('') + os.sep) + '(.*)', '\\1')]
        self.passthru_includes = None
        self.include_once = {}
        self.include_depth = 0
        self.include_times = []  # list of FileInclusionTime
        self.return_code = 0
        self.debugout = None
        self.auto_pragma_once_enabled = True
        self.line_directive = '#line'
        self.compress = False
        self.assume_encoding = None

        # Probe the lexer for selected tokens
        self.__lexprobe()

        tm = time.localtime()
        self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm))
        self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm))
        self.define("__PCPP__ 1")
        self.expand_linemacro = True
        self.expand_filemacro = True
        self.expand_countermacro = True
        self.linemacro = 0
        self.linemacrodepth = 0
        self.countermacro = 0
        self.parser = None

    # -----------------------------------------------------------------------------
    # tokenize()
    #
    # Utility function. Given a string of text, tokenize into a list of tokens
    # -----------------------------------------------------------------------------

    def tokenize(self,text):
        """Utility function. Given a string of text, tokenize into a list of tokens"""
        tokens = []
        self.lexer.input(text)
        while True:
            tok = self.lexer.token()
            if not tok: break
            tok.source = ''
            tokens.append(tok)
        return tokens

    # ----------------------------------------------------------------------
    # __lexprobe()
    #
    # This method probes the preprocessor lexer object to discover
    # the token types of symbols that are important to the preprocessor.
    # If this works right, the preprocessor will simply "work"
    # with any suitable lexer regardless of how tokens have been named.
    # ----------------------------------------------------------------------

    def __lexprobe(self):

        # Determine the token type for identifiers
        self.lexer.input("identifier")
        tok = self.lexer.token()
        if not tok or tok.value != "identifier":
            print("Couldn't determine identifier type")
        else:
            self.t_ID = tok.type

        # Determine the token type for integers
        self.lexer.input("12345")
        tok = self.lexer.token()
        if not tok or int(tok.value) != 12345:
            print("Couldn't determine integer type")
        else:
            self.t_INTEGER = tok.type
            self.t_INTEGER_TYPE = type(tok.value)

        # Determine the token type for character
        self.lexer.input("'a'")
        tok = self.lexer.token()
        if not tok or tok.value != "'a'":
            print("Couldn't determine character type")
        else:
            self.t_CHAR = tok.type
            
        # Determine the token type for strings enclosed in double quotes
        self.lexer.input("\"filename\"")
        tok = self.lexer.token()
        if not tok or tok.value != "\"filename\"":
            print("Couldn't determine string type")
        else:
            self.t_STRING = tok.type

        # Determine the token type for whitespace--if any
        self.lexer.input("  ")
        tok = self.lexer.token()
        if not tok or tok.value != "  ":
            self.t_SPACE = None
        else:
            self.t_SPACE = tok.type

        # Determine the token type for newlines
        self.lexer.input("\n")
        tok = self.lexer.token()
        if not tok or tok.value != "\n":
            self.t_NEWLINE = None
            print("Couldn't determine token for newlines")
        else:
            self.t_NEWLINE = tok.type

        # Determine the token type for line continuations
        self.lexer.input("\\     \n")
        tok = self.lexer.token()
        if not tok or tok.value != "     ":
            self.t_LINECONT = None
            print("Couldn't determine token for line continuations")
        else:
            self.t_LINECONT = tok.type

        self.t_WS = (self.t_SPACE, self.t_NEWLINE, self.t_LINECONT)

        self.lexer.input("##")
        tok = self.lexer.token()
        if not tok or tok.value != "##":
            print("Couldn't determine token for token pasting operator")
        else:
            self.t_DPOUND = tok.type

        self.lexer.input("?")
        tok = self.lexer.token()
        if not tok or tok.value != "?":
            print("Couldn't determine token for ternary operator")
        else:
            self.t_TERNARY = tok.type

        self.lexer.input(":")
        tok = self.lexer.token()
        if not tok or tok.value != ":":
            print("Couldn't determine token for ternary operator")
        else:
            self.t_COLON = tok.type

        self.lexer.input("/* comment */")
        tok = self.lexer.token()
        if not tok or tok.value != "/* comment */":
            print("Couldn't determine comment type")
        else:
            self.t_COMMENT1 = tok.type

        self.lexer.input("// comment")
        tok = self.lexer.token()
        if not tok or tok.value != "// comment":
            print("Couldn't determine comment type")
        else:
            self.t_COMMENT2 = tok.type
            
        self.t_COMMENT = (self.t_COMMENT1, self.t_COMMENT2)

        # Check for other characters used by the preprocessor
        chars = [ '<','>','#','##','\\','(',')',',','.']
        for c in chars:
            self.lexer.input(c)
            tok = self.lexer.token()
            if not tok or tok.value != c:
                print("Unable to lex '%s' required for preprocessor" % c)

    # ----------------------------------------------------------------------
    # add_path()
    #
    # Adds a search path to the preprocessor.  
    # ----------------------------------------------------------------------

    def add_path(self,path):
        """Adds a search path to the preprocessor. """
        self.path.append(path)
        # If the search path being added is relative, or has a common ancestor to the
        # current working directory, add a rewrite to relativise includes from this
        # search path
        relpath = None
        try:
            relpath = os.path.relpath(path)
        except: pass
        if relpath is not None:
            self.rewrite_paths += [(re.escape(os.path.abspath(path) + os.sep) + '(.*)', os.path.join(relpath, '\\1'))]


    # ----------------------------------------------------------------------
    # group_lines()
    #
    # Given an input string, this function splits it into lines.  Trailing whitespace
    # is removed. This function forms the lowest level of the preprocessor---grouping
    # text into a line-by-line format.
    # ----------------------------------------------------------------------

    def group_lines(self,input,abssource):
        r"""Given an input string, this function splits it into lines.  Trailing whitespace
        is removed. This function forms the lowest level of the preprocessor---grouping
        text into a line-by-line format.
        """
        lex = self.lexer.clone()
        lines = [x.rstrip() for x in input.splitlines()]

        input = "\n".join(lines)
        lex.input(input)
        lex.lineno = 1

        current_line = []
        while True:
            tok = lex.token()
            if not tok:
                break
            tok.source = abssource
            current_line.append(tok)
            if tok.type in self.t_WS and tok.value == '\n':
                yield current_line
                current_line = []

        if current_line:
            nltok = copy.copy(current_line[-1])
            nltok.type = self.t_NEWLINE
            nltok.value = '\n'
            current_line.append(nltok)
            yield current_line

    # ----------------------------------------------------------------------
    # tokenstrip()
    # 
    # Remove leading/trailing whitespace tokens from a token list
    # ----------------------------------------------------------------------

    def tokenstrip(self,tokens):
        """Remove leading/trailing whitespace tokens from a token list"""
        i = 0
        while i < len(tokens) and tokens[i].type in self.t_WS:
            i += 1
        del tokens[:i]
        i = len(tokens)-1
        while i >= 0 and tokens[i].type in self.t_WS:
            i -= 1
        del tokens[i+1:]
        return tokens


    # ----------------------------------------------------------------------
    # collect_args()
    #
    # Collects comma separated arguments from a list of tokens.   The arguments
    # must be enclosed in parenthesis.  Returns a tuple (tokencount,args,positions)
    # where tokencount is the number of tokens consumed, args is a list of arguments,
    # and positions is a list of integers containing the starting index of each
    # argument.  Each argument is represented by a list of tokens.
    #
    # When collecting arguments, leading and trailing whitespace is removed
    # from each argument.  
    #
    # This function properly handles nested parenthesis and commas---these do not
    # define new arguments.
    # ----------------------------------------------------------------------

    def collect_args(self,tokenlist,ignore_errors=False):
        """Collects comma separated arguments from a list of tokens.   The arguments
        must be enclosed in parenthesis.  Returns a tuple (tokencount,args,positions)
        where tokencount is the number of tokens consumed, args is a list of arguments,
        and positions is a list of integers containing the starting index of each
        argument.  Each argument is represented by a list of tokens.
        
        When collecting arguments, leading and trailing whitespace is removed
        from each argument.  
        
        This function properly handles nested parenthesis and commas---these do not
        define new arguments."""
        args = []
        positions = []
        current_arg = []
        nesting = 1
        tokenlen = len(tokenlist)
    
        # Search for the opening '('.
        i = 0
        while (i < tokenlen) and (tokenlist[i].type in self.t_WS):
            i += 1

        if (i < tokenlen) and (tokenlist[i].value == '('):
            positions.append(i+1)
        else:
            if not ignore_errors:
                self.on_error(tokenlist[0].source,tokenlist[0].lineno,"Missing '(' in macro arguments")
            return 0, [], []

        i += 1

        while i < tokenlen:
            t = tokenlist[i]
            if t.value == '(':
                current_arg.append(t)
                nesting += 1
            elif t.value == ')':
                nesting -= 1
                if nesting == 0:
                    args.append(self.tokenstrip(current_arg))
                    positions.append(i)
                    return i+1,args,positions
                current_arg.append(t)
            elif t.value == ',' and nesting == 1:
                args.append(self.tokenstrip(current_arg))
                positions.append(i+1)
                current_arg = []
            else:
                current_arg.append(t)
            i += 1
    
        # Missing end argument
        if not ignore_errors:
            self.on_error(tokenlist[-1].source,tokenlist[-1].lineno,"Missing ')' in macro arguments")
        return 0, [],[]

    # ----------------------------------------------------------------------
    # macro_prescan()
    #
    # Examine the macro value (token sequence) and identify patch points
    # This is used to speed up macro expansion later on---we'll know
    # right away where to apply patches to the value to form the expansion
    # ----------------------------------------------------------------------
    
    def macro_prescan(self,macro):
        """Examine the macro value (token sequence) and identify patch points
        This is used to speed up macro expansion later on---we'll know
        right away where to apply patches to the value to form the expansion"""
        macro.patch     = []             # Standard macro arguments 
        macro.str_patch = []             # String conversion expansion
        macro.var_comma_patch = []       # Variadic macro comma patch
        i = 0
        #print("BEFORE", macro.value)
        #print("BEFORE", [x.value for x in macro.value])
        while i < len(macro.value):
            if macro.value[i].type == self.t_ID and macro.value[i].value in macro.arglist:
                argnum = macro.arglist.index(macro.value[i].value)
                # Conversion of argument to a string
                j = i - 1
                while j >= 0 and macro.value[j].type in self.t_WS:
                    j -= 1
                if j >= 0 and macro.value[j].value == '#':
                    macro.value[i] = copy.copy(macro.value[i])
                    macro.value[i].type = self.t_STRING
                    while i > j:
                        del macro.value[j]
                        i -= 1
                    macro.str_patch.append((argnum,i))
                    continue
                # Concatenation
                elif (i > 0 and macro.value[i-1].value == '##'):
                    macro.patch.append(('t',argnum,i))
                    i += 1
                    continue
                elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'):
                    macro.patch.append(('t',argnum,i))
                    i += 1
                    continue
                # Standard expansion
                else:
                    macro.patch.append(('e',argnum,i))
            elif macro.value[i].value == '##':
                if macro.variadic and (i > 0) and (macro.value[i-1].value == ',') and \
                        ((i+1) < len(macro.value)) and (macro.value[i+1].type == self.t_ID) and \
                        (macro.value[i+1].value == macro.vararg):
                    macro.var_comma_patch.append(i-1)
            i += 1
        macro.patch.sort(key=lambda x: x[2],reverse=True)
        #print("AFTER", macro.value)
        #print("AFTER", [x.value for x in macro.value])
        #print(macro.patch)

    # ----------------------------------------------------------------------
    # macro_expand_args()
    #
    # Given a Macro and list of arguments (each a token list), this method
    # returns an expanded version of a macro.  The return value is a token sequence
    # representing the replacement macro tokens
    # ----------------------------------------------------------------------

    def macro_expand_args(self,macro,args):
        """Given a Macro and list of arguments (each a token list), this method
        returns an expanded version of a macro.  The return value is a token sequence
        representing the replacement macro tokens"""
        # Make a copy of the macro token sequence
        rep = [copy.copy(_x) for _x in macro.value]

        # Make string expansion patches.  These do not alter the length of the replacement sequence
        str_expansion = {}
        for argnum, i in macro.str_patch:
            if argnum not in str_expansion:
                # Strip all non-space whitespace before stringization
                tokens = copy.copy(args[argnum])
                for j in xrange(len(tokens)):
                    if tokens[j].type in self.t_WS and tokens[j].type != self.t_LINECONT:
                        tokens[j].value = ' '
                # Collapse all multiple whitespace too
                j = 0
                while j < len(tokens) - 1:
                    if tokens[j].type in self.t_WS and tokens[j+1].type in self.t_WS:
                        del tokens[j+1]
                    else:
                        j += 1
                str = "".join([x.value for x in tokens])
                str = str.replace("\\","\\\\").replace('"', '\\"')
                str_expansion[argnum] = '"' + str + '"'
            rep[i] = copy.copy(rep[i])
            rep[i].value = str_expansion[argnum]

        # Make the variadic macro comma patch.  If the variadic macro argument is empty, we get rid
        comma_patch = False
        if macro.variadic and not args[-1]:
            for i in macro.var_comma_patch:
                rep[i] = None
                comma_patch = True

        # Make all other patches.   The order of these matters.  It is assumed that the patch list
        # has been sorted in reverse order of patch location since replacements will cause the
        # size of the replacement sequence to expand from the patch point.
        
        expanded = { }
        #print("***", macro)
        #print(macro.patch)
        for ptype, argnum, i in macro.patch:
            #print([x.value for x in rep])
            # Concatenation.   Argument is left unexpanded
            if ptype == 't':
                rep[i:i+1] = args[argnum]
            # Normal expansion.  Argument is macro expanded first
            elif ptype == 'e':
                #print('*** Function macro arg', rep[i], 'replace with', args[argnum], 'which expands into', self.expand_macros(copy.copy(args[argnum])))
                if argnum not in expanded:
                    expanded[argnum] = self.expand_macros(copy.copy(args[argnum]))
                rep[i:i+1] = expanded[argnum]

        # Get rid of removed comma if necessary
        if comma_patch:
            rep = [_i for _i in rep if _i]
            
        # Do a token concatenation pass, stitching any tokens separated by ## into a single token
        while len(rep) and rep[0].type == self.t_DPOUND:
            del rep[0]
        while len(rep) and rep[-1].type == self.t_DPOUND:
            del rep[-1]
        i = 1
        stitched = False
        while i < len(rep) - 1:
            if rep[i].type == self.t_DPOUND:
                j = i + 1
                while rep[j].type == self.t_DPOUND:
                    j += 1
                rep[i-1] = copy.copy(rep[i-1])
                rep[i-1].type = None
                rep[i-1].value += rep[j].value
                while j >= i:
                    del rep[i]
                    j -= 1
                stitched = True
            else:
                i += 1
        if stitched:
            # Stitched tokens will have unknown type, so figure those out now
            i = 0
            lex = self.lexer.clone()
            while i < len(rep):
                if rep[i].type is None:
                    lex.input(rep[i].value)
                    toks = []
                    while True:
                        tok = lex.token()
                        if not tok:
                            break
                        toks.append(tok)
                    if len(toks) != 1:
                        # Split it once again
                        while len(toks) > 1:
                            rep.insert(i+1, copy.copy(rep[i]))
                            rep[i+1].value = toks[-1].value
                            rep[i+1].type = toks[-1].type
                            toks.pop()
                        rep[i].value = toks[0].value
                        rep[i].type = toks[0].type
                    else:
                        rep[i].type = toks[0].type
                i += 1

        #print rep
        return rep


    # ----------------------------------------------------------------------
    # expand_macros()
    #
    # Given a list of tokens, this function performs macro expansion.
    # ----------------------------------------------------------------------

    def expand_macros(self,tokens,expanding_from=[]):
        """Given a list of tokens, this function performs macro expansion."""
        # Each token needs to track from which macros it has been expanded from to prevent recursion
        for tok in tokens:
            if not hasattr(tok, 'expanded_from'):
                tok.expanded_from = []
        i = 0
        #print("*** EXPAND MACROS in", "".join([t.value for t in tokens]), "expanding_from=", expanding_from)
        #print(tokens)
        #print([(t.value, t.expanded_from) for t in tokens])
        while i < len(tokens):
            t = tokens[i]
            if self.linemacrodepth == 0:
                self.linemacro = t.lineno
            self.linemacrodepth = self.linemacrodepth + 1
            if t.type == self.t_ID:
                if t.value in self.macros and t.value not in t.expanded_from and t.value not in expanding_from:
                    # Yes, we found a macro match
                    m = self.macros[t.value]
                    if m.arglist is None:
                        # A simple macro
                        rep = [copy.copy(_x) for _x in m.value]
                        ex = self.expand_macros(rep, expanding_from + [t.value])
                        #print("\nExpanding macro", m, "\ninto", ex, "\nreplacing", tokens[i:i+1])
                        for e in ex:
                            e.source = t.source
                            e.lineno = t.lineno
                            if not hasattr(e, 'expanded_from'):
                                e.expanded_from = []
                            e.expanded_from.append(t.value)
                        tokens[i:i+1] = ex
                    else:
                        # A macro with arguments
                        j = i + 1
                        while j < len(tokens) and (tokens[j].type in self.t_WS or tokens[j].type in self.t_COMMENT):
                            j += 1
                        # A function like macro without an invocation list is to be ignored
                        if j == len(tokens) or tokens[j].value != '(':
                            i = j
                        else:
                            tokcount,args,positions = self.collect_args(tokens[j:], True)
                            if tokcount == 0:
                                # Unclosed parameter list, just bail out
                                break
                            if (not m.variadic
                                # A no arg or single arg consuming macro is permitted to be expanded with nothing
                                and (args != [[]] or len(m.arglist) > 1)
                                and len(args) !=  len(m.arglist)):
                                self.on_error(t.source,t.lineno,"Macro %s requires %d arguments but was passed %d" % (t.value,len(m.arglist),len(args)))
                                i = j + tokcount
                            elif m.variadic and len(args) < len(m.arglist)-1:
                                if len(m.arglist) > 2:
                                    self.on_error(t.source,t.lineno,"Macro %s must have at least %d arguments" % (t.value, len(m.arglist)-1))
                                else:
                                    self.on_error(t.source,t.lineno,"Macro %s must have at least %d argument" % (t.value, len(m.arglist)-1))
                                i = j + tokcount
                            else:
                                if m.variadic:
                                    if len(args) == len(m.arglist)-1:
                                        args.append([])
                                    else:
                                        args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1]
                                        del args[len(m.arglist):]
                                else:
                                    # If we called a single arg macro with empty, fake extend args
                                    while len(args) < len(m.arglist):
                                        args.append([])
                                        
                                # Get macro replacement text
                                rep = self.macro_expand_args(m,args)
                                ex = self.expand_macros(rep, expanding_from + [t.value])
                                for e in ex:
                                    e.source = t.source
                                    e.lineno = t.lineno
                                    if not hasattr(e, 'expanded_from'):
                                        e.expanded_from = []
                                    e.expanded_from.append(t.value)
                                # A non-conforming extension implemented by the GCC and clang preprocessors
                                # is that an expansion of a macro with arguments where the following token is
                                # an identifier inserts a space between the expansion and the identifier. This
                                # differs from Boost.Wave incidentally (see https://github.com/ned14/pcpp/issues/29)
                                if len(tokens) > j+tokcount and tokens[j+tokcount].type in self.t_ID:
                                    #print("*** token after expansion is", tokens[j+tokcount])
                                    newtok = copy.copy(tokens[j+tokcount])
                                    newtok.type = self.t_SPACE
                                    newtok.value = ' '
                                    ex.append(newtok)
                                #print("\nExpanding macro", m, "\n\ninto", ex, "\n\nreplacing", tokens[i:j+tokcount])
                                tokens[i:j+tokcount] = ex
                    self.linemacrodepth = self.linemacrodepth - 1
                    if self.linemacrodepth == 0:
                        self.linemacro = 0
                    continue
                elif self.expand_linemacro and t.value == '__LINE__':
                    t.type = self.t_INTEGER
                    t.value = self.t_INTEGER_TYPE(self.linemacro)
                elif self.expand_countermacro and t.value == '__COUNTER__':
                    t.type = self.t_INTEGER
                    t.value = self.t_INTEGER_TYPE(self.countermacro)
                    self.countermacro += 1
                
            i += 1
            self.linemacrodepth = self.linemacrodepth - 1
            if self.linemacrodepth == 0:
                self.linemacro = 0
        return tokens

    # ----------------------------------------------------------------------    
    # evalexpr()
    # 
    # Evaluate an expression token sequence for the purposes of evaluating
    # integral expressions.
    # ----------------------------------------------------------------------

    def evalexpr(self,tokens):
        """Evaluate an expression token sequence for the purposes of evaluating
        integral expressions."""
        if not tokens:
            self.on_error('unknown', 0, "Empty expression")
            return (0, None)
        # tokens = tokenize(line)
        # Search for defined macros
        partial_expansion = False
        def replace_defined(tokens):
            i = 0
            while i < len(tokens):
                if tokens[i].type == self.t_ID and tokens[i].value == 'defined':
                    j = i + 1
                    needparen = False
                    result = "0L"
                    while j < len(tokens):
                        if tokens[j].type in self.t_WS:
                            j += 1
                            continue
                        elif tokens[j].type == self.t_ID:
                            if tokens[j].value in self.macros:
                                result = "1L"
                            else:
                                repl = self.on_unknown_macro_in_defined_expr(tokens[j])
                                if repl is None:
                                    partial_expansion = True
                                    result = 'defined('+tokens[j].value+')'
                                else:
                                    result = "1L" if repl else "0L"
                            if not needparen: break
                        elif tokens[j].value == '(':
                            needparen = True
                        elif tokens[j].value == ')':
                            break
                        else:
                            self.on_error(tokens[i].source,tokens[i].lineno,"Malformed defined()")
                        j += 1
                    if result.startswith('defined'):
                        tokens[i].type = self.t_ID
                        tokens[i].value = result
                    else:
                        tokens[i].type = self.t_INTEGER
                        tokens[i].value = self.t_INTEGER_TYPE(result)
                    del tokens[i+1:j+1]
                i += 1
            return tokens
        # Replace any defined(macro) before macro expansion
        tokens = replace_defined(tokens)
        tokens = self.expand_macros(tokens)
        # Replace any defined(macro) after macro expansion
        tokens = replace_defined(tokens)
        if not tokens:
            return (0, None)
        class IndirectToMacroHook(object):
            def __init__(self, p):
                self.__preprocessor = p
                self.partial_expansion = False
            def __contains__(self, key):
                return True
            def __getitem__(self, key):
                if key.startswith('defined('):
                    self.partial_expansion = True
                    return 0
                repl = self.__preprocessor.on_unknown_macro_in_expr(key)
                #print("*** IndirectToMacroHook[", key, "] returns", repl, file = sys.stderr)
                if repl is None:
                    self.partial_expansion = True
                    return key
                return repl
        evalvars = IndirectToMacroHook(self)
        class IndirectToMacroFunctionHook(object):
            def __init__(self, p):
                self.__preprocessor = p
                self.partial_expansion = False
            def __contains__(self, key):
                return True
            def __getitem__(self, key):
                repl = self.__preprocessor.on_unknown_macro_function_in_expr(key)
                #print("*** IndirectToMacroFunctionHook[", key, "] returns", repl, file = sys.stderr)
                if repl is None:
                    self.partial_expansion = True
                    return key
                return repl
        evalfuncts = IndirectToMacroFunctionHook(self)
        try:
            result = self.evaluator(tokens, functions = evalfuncts, identifiers = evalvars).value()
            partial_expansion = partial_expansion or evalvars.partial_expansion or evalfuncts.partial_expansion
        except OutputDirective:
            raise
        except Exception as e:
            partial_expansion = partial_expansion or evalvars.partial_expansion or evalfuncts.partial_expansion
            if not partial_expansion:
                self.on_error(tokens[0].source,tokens[0].lineno,"Could not evaluate expression due to %s (passed to evaluator: '%s')" % (repr(e), ''.join([tok.value for tok in tokens])))
            result = 0
        return (result, tokens) if partial_expansion else (result, None)

    # ----------------------------------------------------------------------
    # parsegen()
    #
    # Parse an input string
    # ----------------------------------------------------------------------
    def parsegen(self,input,source=None,abssource=None):
        """Parse an input string"""
        rewritten_source = source
        if abssource:
            rewritten_source = abssource
            for rewrite in self.rewrite_paths:
                temp = re.sub(rewrite[0], rewrite[1], rewritten_source)
                if temp != abssource:
                    rewritten_source = temp
                    if os.sep != '/':
                        rewritten_source = rewritten_source.replace(os.sep, '/')
                    break

        # Replace trigraph sequences
        t = trigraph(input)
        lines = self.group_lines(t, rewritten_source)

        if not source:
            source = ""
        if not rewritten_source:
            rewritten_source = ""
            
        my_include_times_idx = len(self.include_times)
        self.include_times.append(FileInclusionTime(self.macros['__FILE__'] if '__FILE__' in self.macros else None, source, abssource, self.include_depth))
        self.include_depth += 1
        my_include_time_begin = clock()
        if self.expand_filemacro:
            self.define("__FILE__ \"%s\"" % rewritten_source)

        self.source = abssource
        chunk = []
        enable = True
        iftrigger = False
        ifpassthru = False
        class ifstackentry(object):
            def __init__(self,enable,iftrigger,ifpassthru,startlinetoks):
                self.enable = enable
                self.iftrigger = iftrigger
                self.ifpassthru = ifpassthru
                self.rewritten = False
                self.startlinetoks = startlinetoks
        ifstack = []
        # True until any non-whitespace output or anything with effects happens.
        at_front_of_file = True
        # True if auto pragma once still a possibility for this #include
        auto_pragma_once_possible = self.auto_pragma_once_enabled
        # =(MACRO, 0) means #ifndef MACRO or #if !defined(MACRO) seen, =(MACRO,1) means #define MACRO seen
        include_guard = None
        self.on_potential_include_guard(None)

        for x in lines:
            all_whitespace = True
            skip_auto_pragma_once_possible_check = False
            # Handle comments
            for i,tok in enumerate(x):
                if tok.type in self.t_COMMENT:
                    if not self.on_comment(tok):
                        if tok.type == self.t_COMMENT1:
                            tok.value = ' '
                        elif tok.type == self.t_COMMENT2:
                            tok.value = '\n'
                        tok.type = 'CPP_WS'
            # Skip over whitespace
            for i,tok in enumerate(x):
                if tok.type not in self.t_WS and tok.type not in self.t_COMMENT:
                    all_whitespace = False
                    break
            output_and_expand_line = True
            output_unexpanded_line = False
            if tok.value == '#':
                precedingtoks = [ tok ]
                output_and_expand_line = False
                try:
                    # Preprocessor directive      
                    i += 1
                    while i < len(x) and x[i].type in self.t_WS:
                        precedingtoks.append(x[i])
                        i += 1                    
                    dirtokens = self.tokenstrip(x[i:])
                    if dirtokens:
                        name = dirtokens[0].value
                        args = self.tokenstrip(dirtokens[1:])
                    
                        if self.debugout is not None:
                            print("%d:%d:%d %s:%d #%s %s" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno, dirtokens[0].value, "".join([tok.value for tok in args])), file = self.debugout)
                            #print(ifstack)

                        handling = self.on_directive_handle(dirtokens[0],args,ifpassthru,precedingtoks)
                        assert handling == True or handling == None
                    else:
                        name = ""
                        args = []
                        raise OutputDirective(Action.IgnoreAndRemove)
                        
                    if name == 'define':
                        at_front_of_file = False
                        if enable:
                            for tok in self.expand_macros(chunk):
                                yield tok
                            chunk = []
                            if include_guard and include_guard[1] == 0:
                                if include_guard[0] == args[0].value and len(args) == 1:
                                    include_guard = (args[0].value, 1)
                                    # If ifpassthru is only turned on due to this include guard, turn it off
                                    if ifpassthru and not ifstack[-1].ifpassthru:
                                        ifpassthru = False
                            self.define(args)
                            if self.debugout is not None:
                                print("%d:%d:%d %s:%d      %s" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno, repr(self.macros[args[0].value])), file = self.debugout)
                            if handling is None:
                                for tok in x:
                                    yield tok
                    elif name == 'include':
                        if enable:
                            for tok in self.expand_macros(chunk):
                                yield tok
                            chunk = []
                            oldfile = self.macros['__FILE__'] if '__FILE__' in self.macros else None
                            if args and args[0].value != '<' and args[0].type != self.t_STRING:
                                args = self.tokenstrip(self.expand_macros(args))
                            #print('***', ''.join([x.value for x in args]), file = sys.stderr)
                            if self.passthru_includes is not None and self.passthru_includes.match(''.join([x.value for x in args])):
                                for tok in precedingtoks:
                                    yield tok
                                for tok in dirtokens:
                                    yield tok
                                for tok in self.include(args):
                                    pass
                            else:
                                for tok in self.include(args):
                                    yield tok
                            if oldfile is not None:
                                self.macros['__FILE__'] = oldfile
                            self.source = abssource
                    elif name == 'undef':
                        at_front_of_file = False
                        if enable:
                            for tok in self.expand_macros(chunk):
                                yield tok
                            chunk = []
                            self.undef(args)
                            if handling is None:
                                for tok in x:
                                    yield tok
                    elif name == 'ifdef':
                        at_front_of_file = False
                        ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x))
                        if enable:
                            ifpassthru = False
                            if not args[0].value in self.macros:
                                res = self.on_unknown_macro_in_defined_expr(args[0])
                                if res is None:
                                    ifpassthru = True
                                    ifstack[-1].rewritten = True
                                    raise OutputDirective(Action.IgnoreAndPassThrough)
                                elif res is True:
                                    iftrigger = True
                                else:
                                    enable = False
                                    iftrigger = False
                            else:
                                iftrigger = True
                    elif name == 'ifndef':
                        if not ifstack and at_front_of_file:
                            self.on_potential_include_guard(args[0].value)
                            include_guard = (args[0].value, 0)
                        at_front_of_file = False
                        ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x))
                        if enable:
                            ifpassthru = False
                            if args[0].value in self.macros:
                                enable = False
                                iftrigger = False
                            else:
                                res = self.on_unknown_macro_in_defined_expr(args[0])
                                if res is None:
                                    ifpassthru = True
                                    ifstack[-1].rewritten = True
                                    raise OutputDirective(Action.IgnoreAndPassThrough)
                                elif res is True:
                                    enable = False
                                    iftrigger = False
                                else:
                                    iftrigger = True
                    elif name == 'if':
                        if not ifstack and at_front_of_file:
                            if args[0].value == '!' and args[1].value == 'defined':
                                n = 2
                                if args[n].value == '(': n += 1
                                self.on_potential_include_guard(args[n].value)
                                include_guard = (args[n].value, 0)
                        at_front_of_file = False
                        ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x))
                        if enable:
                            iftrigger = False
                            ifpassthru = False
                            result, rewritten = self.evalexpr(args)
                            if rewritten is not None:
                                x = x[:i+2] + rewritten + [x[-1]]
                                x[i+1] = copy.copy(x[i+1])
                                x[i+1].type = self.t_SPACE
                                x[i+1].value = ' '
                                ifpassthru = True
                                ifstack[-1].rewritten = True
                                raise OutputDirective(Action.IgnoreAndPassThrough)
                            if not result:
                                enable = False
                            else:
                                iftrigger = True
                    elif name == 'elif':
                        at_front_of_file = False
                        if ifstack:
                            if ifstack[-1].enable:     # We only pay attention if outer "if" allows this
                                if enable and not ifpassthru:         # If already true, we flip enable False
                                    enable = False
                                elif not iftrigger:   # If False, but not triggered yet, we'll check expression
                                    result, rewritten = self.evalexpr(args)
                                    if rewritten is not None:
                                        enable = True
                                        if not ifpassthru:
                                            # This is a passthru #elif after a False #if, so convert to an #if
                                            x[i].value = 'if'
                                        x = x[:i+2] + rewritten + [x[-1]]
                                        x[i+1] = copy.copy(x[i+1])
                                        x[i+1].type = self.t_SPACE
                                        x[i+1].value = ' '
                                        ifpassthru = True
                                        ifstack[-1].rewritten = True
                                        raise OutputDirective(Action.IgnoreAndPassThrough)
                                    if ifpassthru:
                                        # If this elif can only ever be true, simulate that
                                        if result:
                                            newtok = copy.copy(x[i+3])
                                            newtok.type = self.t_INTEGER
                                            newtok.value = self.t_INTEGER_TYPE(result)
                                            x = x[:i+2] + [newtok] + [x[-1]]
                                            raise OutputDirective(Action.IgnoreAndPassThrough)
                                        # Otherwise elide
                                        enable = False
                                    elif result:
                                        enable  = True
                                        iftrigger = True
                        else:
                            self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #elif")
                            
                    elif name == 'else':
                        at_front_of_file = False
                        if ifstack:
                            if ifstack[-1].enable:
                                if ifpassthru:
                                    enable = True
                                    raise OutputDirective(Action.IgnoreAndPassThrough)
                                if enable:
                                    enable = False
                                elif not iftrigger:
                                    enable = True
                                    iftrigger = True
                        else:
                            self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #else")

                    elif name == 'endif':
                        at_front_of_file = False
                        if ifstack:
                            oldifstackentry = ifstack.pop()
                            enable = oldifstackentry.enable
                            iftrigger = oldifstackentry.iftrigger
                            ifpassthru = oldifstackentry.ifpassthru
                            if self.debugout is not None:
                                print("%d:%d:%d %s:%d      (%s:%d %s)" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno,
                                    oldifstackentry.startlinetoks[0].source, oldifstackentry.startlinetoks[0].lineno, "".join([n.value for n in oldifstackentry.startlinetoks])), file = self.debugout)
                            skip_auto_pragma_once_possible_check = True
                            if oldifstackentry.rewritten:
                                raise OutputDirective(Action.IgnoreAndPassThrough)
                        else:
                            self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #endif")
                    elif name == 'pragma' and args[0].value == 'once':
                        if enable:
                            self.include_once[self.source] = None
                    elif enable:
                        # Unknown preprocessor directive
                        output_unexpanded_line = (self.on_directive_unknown(dirtokens[0], args, ifpassthru, precedingtoks) is None)

                except OutputDirective as e:
                    if e.action == Action.IgnoreAndPassThrough:
                        output_unexpanded_line = True
                    elif e.action == Action.IgnoreAndRemove:
                        pass
                    else:
                        assert False

            # If there is ever any non-whitespace output outside an include guard, auto pragma once is not possible
            if not skip_auto_pragma_once_possible_check and auto_pragma_once_possible and not ifstack and not all_whitespace:
                auto_pragma_once_possible = False
                if self.debugout is not None:
                    print("%d:%d:%d %s:%d Determined that #include \"%s\" is not entirely wrapped in an include guard macro, disabling auto-applying #pragma once" % (enable, iftrigger, ifpassthru, x[0].source, x[0].lineno, self.source), file = self.debugout)
                
            if output_and_expand_line or output_unexpanded_line:
                if not all_whitespace:
                    at_front_of_file = False

                # Normal text
                if enable:
                    if output_and_expand_line:
                        chunk.extend(x)
                    elif output_unexpanded_line:
                        for tok in self.expand_macros(chunk):
                            yield tok
                        chunk = []
                        for tok in x:
                            yield tok
                else:
                    # Need to extend with the same number of blank lines
                    i = 0
                    while i < len(x):
                        if x[i].type not in self.t_WS:
                            del x[i]
                        else:
                            i += 1
                    chunk.extend(x)

        for tok in self.expand_macros(chunk):
            yield tok
        chunk = []
        for i in ifstack:
            self.on_error(i.startlinetoks[0].source, i.startlinetoks[0].lineno, "Unterminated " + "".join([n.value for n in i.startlinetoks]))
        if auto_pragma_once_possible and include_guard and include_guard[1] == 1:
            if self.debugout is not None:
                print("%d:%d:%d %s:%d Determined that #include \"%s\" is entirely wrapped in an include guard macro called %s, auto-applying #pragma once" % (enable, iftrigger, ifpassthru, self.source, 0, self.source, include_guard[0]), file = self.debugout)
            self.include_once[self.source] = include_guard[0]
        elif self.auto_pragma_once_enabled and self.source not in self.include_once:
            if self.debugout is not None:
                print("%d:%d:%d %s:%d Did not auto apply #pragma once to this file due to auto_pragma_once_possible=%d, include_guard=%s" % (enable, iftrigger, ifpassthru, self.source, 0, auto_pragma_once_possible, repr(include_guard)), file = self.debugout)
        my_include_time_end = clock()
        self.include_times[my_include_times_idx].elapsed = my_include_time_end - my_include_time_begin
        self.include_depth -= 1

    # ----------------------------------------------------------------------
    # include()
    #
    # Implementation of file-inclusion
    # ----------------------------------------------------------------------

    def include(self,tokens):
        """Implementation of file-inclusion"""
        # Try to extract the filename and then process an include file
        if not tokens:
            return
        if tokens:
            if tokens[0].value != '<' and tokens[0].type != self.t_STRING:
                tokens = self.tokenstrip(self.expand_macros(tokens))

            is_system_include = False
            if tokens[0].value == '<':
                is_system_include = True
                # Include <...>
                i = 1
                while i < len(tokens):
                    if tokens[i].value == '>':
                        break
                    i += 1
                else:
                    self.on_error(tokens[0].source,tokens[0].lineno,"Malformed #include <...>")
                    return
                filename = "".join([x.value for x in tokens[1:i]])
                # Search only formally specified paths
                path = self.path
            elif tokens[0].type == self.t_STRING:
                filename = tokens[0].value[1:-1]
                # Search from each nested include file, as well as formally specified paths
                path = self.temp_path + self.path
            else:
                p = self.on_include_not_found(True,False,self.temp_path[0] if self.temp_path else '',tokens[0].value)
                assert p is None
                return
        if not path:
            path = ['']
        while True:
            #print path
            for p in path:
                iname = os.path.join(p,filename)
                fulliname = os.path.abspath(iname)
                if fulliname in self.include_once:
                    if self.debugout is not None:
                        print("x:x:x x:x #include \"%s\" skipped as already seen" % (fulliname), file = self.debugout)
                    return
                try:
                    ih = self.on_file_open(is_system_include,fulliname)
                    data = ih.read()
                    ih.close()
                    dname = os.path.dirname(fulliname)
                    if dname:
                        self.temp_path.insert(0,dname)
                    for tok in self.parsegen(data,filename,fulliname):
                        yield tok
                    if dname:
                        del self.temp_path[0]
                    return
                except IOError:
                    pass
            else:
                p = self.on_include_not_found(False,is_system_include,self.temp_path[0] if self.temp_path else '',filename)
                assert p is not None
                path.append(p)

    # ----------------------------------------------------------------------
    # define()
    #
    # Define a new macro
    # ----------------------------------------------------------------------

    def define(self,tokens):
        """Define a new macro"""
        if isinstance(tokens,STRING_TYPES):
            tokens = self.tokenize(tokens)
        else:
            tokens = [copy.copy(tok) for tok in tokens]
        def add_macro(self, name, macro):
            macro.source = name.source
            macro.lineno = name.lineno
            self.macros[name.value] = macro

        linetok = tokens
        try:
            name = linetok[0]
            if len(linetok) > 1:
                mtype = linetok[1]
            else:
                mtype = None
            if not mtype:
                m = Macro(name.value,[])
                add_macro(self, name, m)
            elif mtype.type in self.t_WS:
                # A normal macro
                m = Macro(name.value,self.tokenstrip(linetok[2:]))
                add_macro(self, name, m)
            elif mtype.value == '(':
                # A macro with arguments
                tokcount, args, positions = self.collect_args(linetok[1:])
                variadic = False
                for a in args:
                    if variadic:
                        self.on_error(name.source,name.lineno,"No more arguments may follow a variadic argument")
                        break
                    astr = "".join([str(_i.value) for _i in a])
                    if astr == "...":
                        variadic = True
                        a[0].type = self.t_ID
                        a[0].value = '__VA_ARGS__'
                        variadic = True
                        del a[1:]
                        continue
                    elif astr[-3:] == "..." and a[0].type == self.t_ID:
                        variadic = True
                        del a[1:]
                        # If, for some reason, "." is part of the identifier, strip off the name for the purposes
                        # of macro expansion
                        if a[0].value[-3:] == '...':
                            a[0].value = a[0].value[:-3]
                        continue
                    # Empty arguments are permitted
                    if len(a) == 0 and len(args) == 1:
                        continue
                    if len(a) > 1 or a[0].type != self.t_ID:
                        self.on_error(a[0].source,a[0].lineno,"Invalid macro argument")
                        break
                else:
                    mvalue = self.tokenstrip(linetok[1+tokcount:])
                    i = 0
                    while i < len(mvalue):
                        if i+1 < len(mvalue):
                            if mvalue[i].type in self.t_WS and mvalue[i+1].value == '##':
                                del mvalue[i]
                                continue
                            elif mvalue[i].value == '##' and mvalue[i+1].type in self.t_WS:
                                del mvalue[i+1]
                        i += 1
                    m = Macro(name.value,mvalue,[x[0].value for x in args] if args != [[]] else [],variadic)
                    self.macro_prescan(m)
                    add_macro(self, name, m)
            else:
                self.on_error(name.source,name.lineno,"Bad macro definition")
        #except LookupError:
        #    print("Bad macro definition")
        except:
            raise

    # ----------------------------------------------------------------------
    # undef()
    #
    # Undefine a macro
    # ----------------------------------------------------------------------

    def undef(self,tokens):
        """Undefine a macro"""
        if isinstance(tokens,STRING_TYPES):
            tokens = self.tokenize(tokens)
        id = tokens[0].value
        try:
            del self.macros[id]
        except LookupError:
            pass

    # ----------------------------------------------------------------------
    # parse()
    #
    # Parse input text.
    # ----------------------------------------------------------------------
    def parse(self,input,source=None,ignore={}):
        """Parse input text."""
        if isinstance(input, FILE_TYPES):
            if source is None:
                source = input.name
            input = input.read()
        self.ignore = ignore
        self.parser = self.parsegen(input,source,os.path.abspath(source) if source else None)
        if source is not None:
            dname = os.path.dirname(source)
            self.temp_path.insert(0,dname)
        
    # ----------------------------------------------------------------------
    # token()
    #
    # Method to return individual tokens
    # ----------------------------------------------------------------------
    def token(self):
        """Method to return individual tokens"""
        try:
            while True:
                tok = next(self.parser)
                if tok.type not in self.ignore:
                    return tok
        except StopIteration:
            self.parser = None
            return None
            
    def write(self, oh=sys.stdout):
        """Calls token() repeatedly, expanding tokens to their text and writing to the file like stream oh"""
        lastlineno = 0
        lastsource = None
        done = False
        blanklines = 0
        while not done:
            emitlinedirective = False
            toks = []
            all_ws = True
            # Accumulate a line
            while not done:
                tok = self.token()
                if not tok:
                    done = True
                    break
                toks.append(tok)
                if tok.value and tok.value[0] == '\n':
                    break
                if tok.type not in self.t_WS:
                    all_ws = False
            if not toks:
                break
            if all_ws:
                # Remove preceding whitespace so it becomes just a LF
                if len(toks) > 1:
                    tok = toks[-1]
                    toks = [ tok ]
                blanklines += toks[0].value.count('\n')
                continue
            # Filter out line continuations, collapsing before and after if needs be
            for n in xrange(len(toks)-1, -1, -1):
                if toks[n].type in self.t_LINECONT:
                    if n > 0 and n < len(toks) - 1 and toks[n-1].type in self.t_WS and toks[n+1].type in self.t_WS:
                        toks[n-1].value = toks[n-1].value[0]
                        del toks[n:n+2]
                    else:
                        del toks[n]
            # The line in toks is not all whitespace
            emitlinedirective = (blanklines > 6) and self.line_directive is not None
            if hasattr(toks[0], 'source'):
                if lastsource is None:
                    if toks[0].source is not None:
                        emitlinedirective = True
                    lastsource = toks[0].source
                elif lastsource != toks[0].source:
                    emitlinedirective = True
                    lastsource = toks[0].source
            # Replace consecutive whitespace in output with a single space except at any indent
            first_ws = None
            #print(toks)
            for n in xrange(len(toks)-1, -1, -1):
                tok = toks[n]
                if first_ws is None:
                    if tok.type in self.t_SPACE or len(tok.value) == 0:
                        first_ws = n
                else:
                    if tok.type not in self.t_SPACE and len(tok.value) > 0:
                        m = n + 1
                        while m != first_ws:
                            del toks[m]
                            first_ws -= 1
                        first_ws = None
                        if self.compress > 0:
                            # Collapse a token of many whitespace into single
                            if toks[m].value and toks[m].value[0] == ' ':
                                toks[m].value = ' '
            if not self.compress > 1 and not emitlinedirective:
                newlinesneeded = toks[0].lineno - lastlineno - 1
                if newlinesneeded > 6 and self.line_directive is not None:
                    emitlinedirective = True
                else:
                    while newlinesneeded > 0:
                        oh.write('\n')
                        newlinesneeded -= 1
            lastlineno = toks[0].lineno
            # Account for those newlines in a multiline comment
            if emitlinedirective and self.line_directive is not None:
                oh.write(self.line_directive + ' ' + str(lastlineno) + ('' if lastsource is None else (' "' + lastsource + '"' )) + '\n')
            for tok in toks:
                if tok.type == self.t_COMMENT1:
                    lastlineno += tok.value.count('\n')
            blanklines = 0
            #print toks[0].lineno, 
            for tok in toks:
                #print tok.value,
                oh.write(tok.value)
            #print ''

if __name__ == "__main__":
    import doctest
    doctest.testmod()

Classes

class Action

What kind of abort processing to do in OutputDirective

Source code
class Action(object):
    """What kind of abort processing to do in OutputDirective"""
    IgnoreAndPassThrough = 0
    """Abort processing (don't execute), but pass the directive through to output"""
    IgnoreAndRemove = 1
    """Abort processing (don't execute), and remove from output"""

Class variables

var IgnoreAndPassThrough

Abort processing (don't execute), but pass the directive through to output

var IgnoreAndRemove

Abort processing (don't execute), and remove from output

class Evaluator

Evaluator of #if C preprocessor expressions.

>>> e = Evaluator()
>>> e('5')
Value(5)
>>> e('5+6')
Value(11)
>>> e('5+6*2')
Value(17)
>>> e('5/2+6*2')
Value(14)
>>> e('5 < 6 <= 7')
Value(1)
>>> e('5 < 6 && 8 > 7')
Value(1)
>>> e('18446744073709551615 == -1')
Value(1)
>>> e('-9223372036854775809 == 9223372036854775807')
Value(1)
>>> e('-1 < 0U')
Value(0U)
>>> e('(( 0L && 0) || (!0L && !0 ))')
Value(1)
>>> e('(1)?2:3')
Value(2)
>>> e('(1 ? -1 : 0) <= 0')
Value(1)
>>> e('(1 ? -1 : 0U)')       # Output type of ? must be common between both choices
Value(18446744073709551615U)
>>> e('(1 ? -1 : 0U) <= 0')
Value(0U)
>>> e('1 && 10 / 0')         # doctest: +ELLIPSIS
Exception(ZeroDivisionError('division by zero'...
>>> e('0 && 10 / 0')         # && must shortcut
Value(0)
>>> e('1 ? 10 / 0 : 0')      # doctest: +ELLIPSIS
Exception(ZeroDivisionError('division by zero'...
>>> e('0 ? 10 / 0 : 0')      # ? must shortcut
Value(0)
>>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
Value(0)
>>> e('1 << 2 != 4 || 8 >> 1 != 4')
Value(0)
>>> e('(2 || 3) != 1 || (2 && 3) != 1 || (0 || 4) != 1 || (0 && 5) != 0')
Value(0)
>>> e('-1 << 3U > 0')
Value(0)
>>> e("'N' == 78")
Value(1)
>>> e('0x3f == 63')
Value(1)
>>> e("'\\n'")
Value(10)
>>> e("'\\\\'")
Value(92)
>>> e("'\\n' == 0xA")
Value(1)
>>> e("'\\\\' == 0x5c")
Value(1)
>>> e("L'\\0' == 0")
Value(1)
>>> e('12 == 12')
Value(1)
>>> e('12L == 12')
Value(1)
>>> e('-1 >= 0U')
Value(1U)
>>> e('(1<<2) == 4')
Value(1)
>>> e('(-!+!9) == -1')
Value(1)
>>> e('(2 || 3) == 1')
Value(1)
>>> e('1L * 3 != 3')
Value(0)
>>> e('(!1L != 0) || (-1L != -1)')
Value(0)
>>> e('0177777 == 65535')
Value(1)
>>> e('0Xffff != 65535 || 0XFfFf == 65535')
Value(1)
>>> e('0L != 0 || 0l != 0')
Value(0)
>>> e('1U != 1 || 1u == 1')
Value(1)
>>> e('0 <= -1')
Value(0)
>>> e('1 << 2 != 4 || 8 >> 1 == 4')
Value(1)
>>> e('(3 ^ 5) == 6')
Value(1)
>>> e('(3 | 5) == 7')
Value(1)
>>> e('(3 & 5) == 1')
Value(1)
>>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
Value(0)
>>> e('(0 ? 1 : 2) != 2')
Value(0)
>>> e('-1 << 3U > 0')
Value(0)
>>> e('0 && 10 / 0')
Value(0)
>>> e('not_defined && 10 / not_defined')  # doctest: +ELLIPSIS
Exception(SyntaxError('Unknown identifier not_defined'...
>>> e('0 && 10 / 0 > 1')
Value(0)
>>> e('(0) ? 10 / 0 : 0')
Value(0)
>>> e('0 == 0 || 10 / 0 > 1')
Value(1)
>>> e('(15 >> 2 >> 1 != 1) || (3 << 2 << 1 != 24)')
Value(0)
>>> e('(1 | 2) == 3 && 4 != 5 || 0')
Value(1)
>>> e('1  >  0')
Value(1)
>>> e("'S' != 83")
Value(0)
>>> e("'' != ''")
Value(0)
>>> e('0 + (1 - (2 + (3 - (4 + (5 - (6 + (7 - (8 + (9 - (10 + (11 - (12 +          (13 - (14 + (15 - (16 + (17 - (18 + (19 - (20 + (21 - (22 + (23 -           (24 + (25 - (26 + (27 - (28 + (29 - (30 + (31 - (32 + 0))))))))))           )))))))))))))))))))))) == 0')
Value(1)
>>> e('test_function(X)', functions={'test_function':lambda x: 55})
Value(55)
>>> e('test_identifier', identifiers={'test_identifier':11})
Value(11)
>>> e('defined(X)', functions={'defined':lambda x: 55})
Value(55)
>>> e('defined(X)')  # doctest: +ELLIPSIS
Exception(SyntaxError('Unknown function defined'...
>>> e('__has_include("variant")')  # doctest: +ELLIPSIS
Exception(SyntaxError('Unknown function __has_include'...
>>> e('__has_include(<variant>)')  # doctest: +ELLIPSIS
Exception(SyntaxError('Unknown function __has_include'...
>>> e('5  // comment')
Value(5)
>>> e('5  /* comment */')
Value(5)
>>> e('5  /* comment // more */')
Value(5)
>>> e('5  // /* comment */')

Value(5)

Source code
class Evaluator(object):
    """Evaluator of #if C preprocessor expressions.
    
    >>> e = Evaluator()
    >>> e('5')
    Value(5)
    >>> e('5+6')
    Value(11)
    >>> e('5+6*2')
    Value(17)
    >>> e('5/2+6*2')
    Value(14)
    >>> e('5 < 6 <= 7')
    Value(1)
    >>> e('5 < 6 && 8 > 7')
    Value(1)
    >>> e('18446744073709551615 == -1')
    Value(1)
    >>> e('-9223372036854775809 == 9223372036854775807')
    Value(1)
    >>> e('-1 < 0U')
    Value(0U)
    >>> e('(( 0L && 0) || (!0L && !0 ))')
    Value(1)
    >>> e('(1)?2:3')
    Value(2)
    >>> e('(1 ? -1 : 0) <= 0')
    Value(1)
    >>> e('(1 ? -1 : 0U)')       # Output type of ? must be common between both choices
    Value(18446744073709551615U)
    >>> e('(1 ? -1 : 0U) <= 0')
    Value(0U)
    >>> e('1 && 10 / 0')         # doctest: +ELLIPSIS
    Exception(ZeroDivisionError('division by zero'...
    >>> e('0 && 10 / 0')         # && must shortcut
    Value(0)
    >>> e('1 ? 10 / 0 : 0')      # doctest: +ELLIPSIS
    Exception(ZeroDivisionError('division by zero'...
    >>> e('0 ? 10 / 0 : 0')      # ? must shortcut
    Value(0)
    >>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
    Value(0)
    >>> e('1 << 2 != 4 || 8 >> 1 != 4')
    Value(0)
    >>> e('(2 || 3) != 1 || (2 && 3) != 1 || (0 || 4) != 1 || (0 && 5) != 0')
    Value(0)
    >>> e('-1 << 3U > 0')
    Value(0)
    >>> e("'N' == 78")
    Value(1)
    >>> e('0x3f == 63')
    Value(1)
    >>> e("'\\\\n'")
    Value(10)
    >>> e("'\\\\\\\\'")
    Value(92)
    >>> e("'\\\\n' == 0xA")
    Value(1)
    >>> e("'\\\\\\\\' == 0x5c")
    Value(1)
    >>> e("L'\\\\0' == 0")
    Value(1)
    >>> e('12 == 12')
    Value(1)
    >>> e('12L == 12')
    Value(1)
    >>> e('-1 >= 0U')
    Value(1U)
    >>> e('(1<<2) == 4')
    Value(1)
    >>> e('(-!+!9) == -1')
    Value(1)
    >>> e('(2 || 3) == 1')
    Value(1)
    >>> e('1L * 3 != 3')
    Value(0)
    >>> e('(!1L != 0) || (-1L != -1)')
    Value(0)
    >>> e('0177777 == 65535')
    Value(1)
    >>> e('0Xffff != 65535 || 0XFfFf == 65535')
    Value(1)
    >>> e('0L != 0 || 0l != 0')
    Value(0)
    >>> e('1U != 1 || 1u == 1')
    Value(1)
    >>> e('0 <= -1')
    Value(0)
    >>> e('1 << 2 != 4 || 8 >> 1 == 4')
    Value(1)
    >>> e('(3 ^ 5) == 6')
    Value(1)
    >>> e('(3 | 5) == 7')
    Value(1)
    >>> e('(3 & 5) == 1')
    Value(1)
    >>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1')
    Value(0)
    >>> e('(0 ? 1 : 2) != 2')
    Value(0)
    >>> e('-1 << 3U > 0')
    Value(0)
    >>> e('0 && 10 / 0')
    Value(0)
    >>> e('not_defined && 10 / not_defined')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown identifier not_defined'...
    >>> e('0 && 10 / 0 > 1')
    Value(0)
    >>> e('(0) ? 10 / 0 : 0')
    Value(0)
    >>> e('0 == 0 || 10 / 0 > 1')
    Value(1)
    >>> e('(15 >> 2 >> 1 != 1) || (3 << 2 << 1 != 24)')
    Value(0)
    >>> e('(1 | 2) == 3 && 4 != 5 || 0')
    Value(1)
    >>> e('1  >  0')
    Value(1)
    >>> e("'\123' != 83")
    Value(0)
    >>> e("'\x1b' != '\033'")
    Value(0)
    >>> e('0 + (1 - (2 + (3 - (4 + (5 - (6 + (7 - (8 + (9 - (10 + (11 - (12 +          (13 - (14 + (15 - (16 + (17 - (18 + (19 - (20 + (21 - (22 + (23 -           (24 + (25 - (26 + (27 - (28 + (29 - (30 + (31 - (32 + 0))))))))))           )))))))))))))))))))))) == 0')
    Value(1)
    >>> e('test_function(X)', functions={'test_function':lambda x: 55})
    Value(55)
    >>> e('test_identifier', identifiers={'test_identifier':11})
    Value(11)
    >>> e('defined(X)', functions={'defined':lambda x: 55})
    Value(55)
    >>> e('defined(X)')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown function defined'...
    >>> e('__has_include("variant")')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown function __has_include'...
    >>> e('__has_include(<variant>)')  # doctest: +ELLIPSIS
    Exception(SyntaxError('Unknown function __has_include'...
    >>> e('5  // comment')
    Value(5)
    >>> e('5  /* comment */')
    Value(5)
    >>> e('5  /* comment // more */')
    Value(5)
    >>> e('5  // /* comment */')
    Value(5)
    """
#    >>> e('defined X', functions={'defined':lambda x: 55})
#    Value(55)

    def __init__(self, lexer = None):
        self.lexer = lexer if lexer is not None else default_lexer()
        self.parser = yacc.yacc(optimize=in_production,debug=not in_production,write_tables=not in_production)

    class __lexer(object):

        def __init__(self, functions, identifiers):
            self.__toks = []
            self.__functions = functions
            self.__identifiers = identifiers

        def input(self, toks):
            self.__toks = [tok for tok in toks if tok.type != 'CPP_WS' and tok.type != 'CPP_LINECONT' and tok.type != 'CPP_COMMENT1' and tok.type != 'CPP_COMMENT2']
            self.__idx = 0

        def token(self):
            if self.__idx >= len(self.__toks):
                return None
            self.__idx = self.__idx + 1
            return self.__toks[self.__idx - 1]

        def on_function_call(self, p):
            if p[1] not in self.__functions:
                raise SyntaxError('Unknown function %s' % p[1])
            p[0] = Value(self.__functions[p[1]](p[3]))

        def on_identifier(self, p):
            if p[1] not in self.__identifiers:
                raise SyntaxError('Unknown identifier %s' % p[1])
            p[0] = Value(self.__identifiers[p[1]])
            
    def __call__(self, input, functions = {}, identifiers = {}):
        """Execute a fully macro expanded set of tokens representing an expression,
        returning the result of the evaluation.
        """
        if not isinstance(input,list):
            self.lexer.input(input)
            input = []
            while True:
                tok = self.lexer.token()
                if not tok:
                    break
                input.append(tok)
        return self.parser.parse(input, lexer = self.__lexer(functions, identifiers))

Methods

def __init__(self, lexer=None)

Initialize self. See help(type(self)) for accurate signature.

Source code
def __init__(self, lexer = None):
    self.lexer = lexer if lexer is not None else default_lexer()
    self.parser = yacc.yacc(optimize=in_production,debug=not in_production,write_tables=not in_production)
class OutputDirective (ancestors: builtins.Exception, builtins.BaseException)

Raise this exception to abort processing of a preprocessor directive and to instead output it as is into the output

Source code
class OutputDirective(Exception):
    """Raise this exception to abort processing of a preprocessor directive and
    to instead output it as is into the output"""
    def __init__(self, action):
        self.action = action

Methods

def __init__(self, action)

Initialize self. See help(type(self)) for accurate signature.

Source code
def __init__(self, action):
    self.action = action
class Preprocessor (ancestors: PreprocessorHooks)

Override these in your subclass of Preprocessor to customise preprocessing

Source code
class Preprocessor(PreprocessorHooks):    
    def __init__(self,lexer=None):
        super(Preprocessor, self).__init__()
        if lexer is None:
            lexer = default_lexer()
        self.lexer = lexer
        self.evaluator = Evaluator(self.lexer)
        self.macros = { }
        self.path = []           # list of -I formal search paths for includes
        self.temp_path = []      # list of temporary search paths for includes
        self.rewrite_paths = [(re.escape(os.path.abspath('') + os.sep) + '(.*)', '\\1')]
        self.passthru_includes = None
        self.include_once = {}
        self.include_depth = 0
        self.include_times = []  # list of FileInclusionTime
        self.return_code = 0
        self.debugout = None
        self.auto_pragma_once_enabled = True
        self.line_directive = '#line'
        self.compress = False
        self.assume_encoding = None

        # Probe the lexer for selected tokens
        self.__lexprobe()

        tm = time.localtime()
        self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm))
        self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm))
        self.define("__PCPP__ 1")
        self.expand_linemacro = True
        self.expand_filemacro = True
        self.expand_countermacro = True
        self.linemacro = 0
        self.linemacrodepth = 0
        self.countermacro = 0
        self.parser = None

    # -----------------------------------------------------------------------------
    # tokenize()
    #
    # Utility function. Given a string of text, tokenize into a list of tokens
    # -----------------------------------------------------------------------------

    def tokenize(self,text):
        """Utility function. Given a string of text, tokenize into a list of tokens"""
        tokens = []
        self.lexer.input(text)
        while True:
            tok = self.lexer.token()
            if not tok: break
            tok.source = ''
            tokens.append(tok)
        return tokens

    # ----------------------------------------------------------------------
    # __lexprobe()
    #
    # This method probes the preprocessor lexer object to discover
    # the token types of symbols that are important to the preprocessor.
    # If this works right, the preprocessor will simply "work"
    # with any suitable lexer regardless of how tokens have been named.
    # ----------------------------------------------------------------------

    def __lexprobe(self):

        # Determine the token type for identifiers
        self.lexer.input("identifier")
        tok = self.lexer.token()
        if not tok or tok.value != "identifier":
            print("Couldn't determine identifier type")
        else:
            self.t_ID = tok.type

        # Determine the token type for integers
        self.lexer.input("12345")
        tok = self.lexer.token()
        if not tok or int(tok.value) != 12345:
            print("Couldn't determine integer type")
        else:
            self.t_INTEGER = tok.type
            self.t_INTEGER_TYPE = type(tok.value)

        # Determine the token type for character
        self.lexer.input("'a'")
        tok = self.lexer.token()
        if not tok or tok.value != "'a'":
            print("Couldn't determine character type")
        else:
            self.t_CHAR = tok.type
            
        # Determine the token type for strings enclosed in double quotes
        self.lexer.input("\"filename\"")
        tok = self.lexer.token()
        if not tok or tok.value != "\"filename\"":
            print("Couldn't determine string type")
        else:
            self.t_STRING = tok.type

        # Determine the token type for whitespace--if any
        self.lexer.input("  ")
        tok = self.lexer.token()
        if not tok or tok.value != "  ":
            self.t_SPACE = None
        else:
            self.t_SPACE = tok.type

        # Determine the token type for newlines
        self.lexer.input("\n")
        tok = self.lexer.token()
        if not tok or tok.value != "\n":
            self.t_NEWLINE = None
            print("Couldn't determine token for newlines")
        else:
            self.t_NEWLINE = tok.type

        # Determine the token type for line continuations
        self.lexer.input("\\     \n")
        tok = self.lexer.token()
        if not tok or tok.value != "     ":
            self.t_LINECONT = None
            print("Couldn't determine token for line continuations")
        else:
            self.t_LINECONT = tok.type

        self.t_WS = (self.t_SPACE, self.t_NEWLINE, self.t_LINECONT)

        self.lexer.input("##")
        tok = self.lexer.token()
        if not tok or tok.value != "##":
            print("Couldn't determine token for token pasting operator")
        else:
            self.t_DPOUND = tok.type

        self.lexer.input("?")
        tok = self.lexer.token()
        if not tok or tok.value != "?":
            print("Couldn't determine token for ternary operator")
        else:
            self.t_TERNARY = tok.type

        self.lexer.input(":")
        tok = self.lexer.token()
        if not tok or tok.value != ":":
            print("Couldn't determine token for ternary operator")
        else:
            self.t_COLON = tok.type

        self.lexer.input("/* comment */")
        tok = self.lexer.token()
        if not tok or tok.value != "/* comment */":
            print("Couldn't determine comment type")
        else:
            self.t_COMMENT1 = tok.type

        self.lexer.input("// comment")
        tok = self.lexer.token()
        if not tok or tok.value != "// comment":
            print("Couldn't determine comment type")
        else:
            self.t_COMMENT2 = tok.type
            
        self.t_COMMENT = (self.t_COMMENT1, self.t_COMMENT2)

        # Check for other characters used by the preprocessor
        chars = [ '<','>','#','##','\\','(',')',',','.']
        for c in chars:
            self.lexer.input(c)
            tok = self.lexer.token()
            if not tok or tok.value != c:
                print("Unable to lex '%s' required for preprocessor" % c)

    # ----------------------------------------------------------------------
    # add_path()
    #
    # Adds a search path to the preprocessor.  
    # ----------------------------------------------------------------------

    def add_path(self,path):
        """Adds a search path to the preprocessor. """
        self.path.append(path)
        # If the search path being added is relative, or has a common ancestor to the
        # current working directory, add a rewrite to relativise includes from this
        # search path
        relpath = None
        try:
            relpath = os.path.relpath(path)
        except: pass
        if relpath is not None:
            self.rewrite_paths += [(re.escape(os.path.abspath(path) + os.sep) + '(.*)', os.path.join(relpath, '\\1'))]


    # ----------------------------------------------------------------------
    # group_lines()
    #
    # Given an input string, this function splits it into lines.  Trailing whitespace
    # is removed. This function forms the lowest level of the preprocessor---grouping
    # text into a line-by-line format.
    # ----------------------------------------------------------------------

    def group_lines(self,input,abssource):
        r"""Given an input string, this function splits it into lines.  Trailing whitespace
        is removed. This function forms the lowest level of the preprocessor---grouping
        text into a line-by-line format.
        """
        lex = self.lexer.clone()
        lines = [x.rstrip() for x in input.splitlines()]

        input = "\n".join(lines)
        lex.input(input)
        lex.lineno = 1

        current_line = []
        while True:
            tok = lex.token()
            if not tok:
                break
            tok.source = abssource
            current_line.append(tok)
            if tok.type in self.t_WS and tok.value == '\n':
                yield current_line
                current_line = []

        if current_line:
            nltok = copy.copy(current_line[-1])
            nltok.type = self.t_NEWLINE
            nltok.value = '\n'
            current_line.append(nltok)
            yield current_line

    # ----------------------------------------------------------------------
    # tokenstrip()
    # 
    # Remove leading/trailing whitespace tokens from a token list
    # ----------------------------------------------------------------------

    def tokenstrip(self,tokens):
        """Remove leading/trailing whitespace tokens from a token list"""
        i = 0
        while i < len(tokens) and tokens[i].type in self.t_WS:
            i += 1
        del tokens[:i]
        i = len(tokens)-1
        while i >= 0 and tokens[i].type in self.t_WS:
            i -= 1
        del tokens[i+1:]
        return tokens


    # ----------------------------------------------------------------------
    # collect_args()
    #
    # Collects comma separated arguments from a list of tokens.   The arguments
    # must be enclosed in parenthesis.  Returns a tuple (tokencount,args,positions)
    # where tokencount is the number of tokens consumed, args is a list of arguments,
    # and positions is a list of integers containing the starting index of each
    # argument.  Each argument is represented by a list of tokens.
    #
    # When collecting arguments, leading and trailing whitespace is removed
    # from each argument.  
    #
    # This function properly handles nested parenthesis and commas---these do not
    # define new arguments.
    # ----------------------------------------------------------------------

    def collect_args(self,tokenlist,ignore_errors=False):
        """Collects comma separated arguments from a list of tokens.   The arguments
        must be enclosed in parenthesis.  Returns a tuple (tokencount,args,positions)
        where tokencount is the number of tokens consumed, args is a list of arguments,
        and positions is a list of integers containing the starting index of each
        argument.  Each argument is represented by a list of tokens.
        
        When collecting arguments, leading and trailing whitespace is removed
        from each argument.  
        
        This function properly handles nested parenthesis and commas---these do not
        define new arguments."""
        args = []
        positions = []
        current_arg = []
        nesting = 1
        tokenlen = len(tokenlist)
    
        # Search for the opening '('.
        i = 0
        while (i < tokenlen) and (tokenlist[i].type in self.t_WS):
            i += 1

        if (i < tokenlen) and (tokenlist[i].value == '('):
            positions.append(i+1)
        else:
            if not ignore_errors:
                self.on_error(tokenlist[0].source,tokenlist[0].lineno,"Missing '(' in macro arguments")
            return 0, [], []

        i += 1

        while i < tokenlen:
            t = tokenlist[i]
            if t.value == '(':
                current_arg.append(t)
                nesting += 1
            elif t.value == ')':
                nesting -= 1
                if nesting == 0:
                    args.append(self.tokenstrip(current_arg))
                    positions.append(i)
                    return i+1,args,positions
                current_arg.append(t)
            elif t.value == ',' and nesting == 1:
                args.append(self.tokenstrip(current_arg))
                positions.append(i+1)
                current_arg = []
            else:
                current_arg.append(t)
            i += 1
    
        # Missing end argument
        if not ignore_errors:
            self.on_error(tokenlist[-1].source,tokenlist[-1].lineno,"Missing ')' in macro arguments")
        return 0, [],[]

    # ----------------------------------------------------------------------
    # macro_prescan()
    #
    # Examine the macro value (token sequence) and identify patch points
    # This is used to speed up macro expansion later on---we'll know
    # right away where to apply patches to the value to form the expansion
    # ----------------------------------------------------------------------
    
    def macro_prescan(self,macro):
        """Examine the macro value (token sequence) and identify patch points
        This is used to speed up macro expansion later on---we'll know
        right away where to apply patches to the value to form the expansion"""
        macro.patch     = []             # Standard macro arguments 
        macro.str_patch = []             # String conversion expansion
        macro.var_comma_patch = []       # Variadic macro comma patch
        i = 0
        #print("BEFORE", macro.value)
        #print("BEFORE", [x.value for x in macro.value])
        while i < len(macro.value):
            if macro.value[i].type == self.t_ID and macro.value[i].value in macro.arglist:
                argnum = macro.arglist.index(macro.value[i].value)
                # Conversion of argument to a string
                j = i - 1
                while j >= 0 and macro.value[j].type in self.t_WS:
                    j -= 1
                if j >= 0 and macro.value[j].value == '#':
                    macro.value[i] = copy.copy(macro.value[i])
                    macro.value[i].type = self.t_STRING
                    while i > j:
                        del macro.value[j]
                        i -= 1
                    macro.str_patch.append((argnum,i))
                    continue
                # Concatenation
                elif (i > 0 and macro.value[i-1].value == '##'):
                    macro.patch.append(('t',argnum,i))
                    i += 1
                    continue
                elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'):
                    macro.patch.append(('t',argnum,i))
                    i += 1
                    continue
                # Standard expansion
                else:
                    macro.patch.append(('e',argnum,i))
            elif macro.value[i].value == '##':
                if macro.variadic and (i > 0) and (macro.value[i-1].value == ',') and \
                        ((i+1) < len(macro.value)) and (macro.value[i+1].type == self.t_ID) and \
                        (macro.value[i+1].value == macro.vararg):
                    macro.var_comma_patch.append(i-1)
            i += 1
        macro.patch.sort(key=lambda x: x[2],reverse=True)
        #print("AFTER", macro.value)
        #print("AFTER", [x.value for x in macro.value])
        #print(macro.patch)

    # ----------------------------------------------------------------------
    # macro_expand_args()
    #
    # Given a Macro and list of arguments (each a token list), this method
    # returns an expanded version of a macro.  The return value is a token sequence
    # representing the replacement macro tokens
    # ----------------------------------------------------------------------

    def macro_expand_args(self,macro,args):
        """Given a Macro and list of arguments (each a token list), this method
        returns an expanded version of a macro.  The return value is a token sequence
        representing the replacement macro tokens"""
        # Make a copy of the macro token sequence
        rep = [copy.copy(_x) for _x in macro.value]

        # Make string expansion patches.  These do not alter the length of the replacement sequence
        str_expansion = {}
        for argnum, i in macro.str_patch:
            if argnum not in str_expansion:
                # Strip all non-space whitespace before stringization
                tokens = copy.copy(args[argnum])
                for j in xrange(len(tokens)):
                    if tokens[j].type in self.t_WS and tokens[j].type != self.t_LINECONT:
                        tokens[j].value = ' '
                # Collapse all multiple whitespace too
                j = 0
                while j < len(tokens) - 1:
                    if tokens[j].type in self.t_WS and tokens[j+1].type in self.t_WS:
                        del tokens[j+1]
                    else:
                        j += 1
                str = "".join([x.value for x in tokens])
                str = str.replace("\\","\\\\").replace('"', '\\"')
                str_expansion[argnum] = '"' + str + '"'
            rep[i] = copy.copy(rep[i])
            rep[i].value = str_expansion[argnum]

        # Make the variadic macro comma patch.  If the variadic macro argument is empty, we get rid
        comma_patch = False
        if macro.variadic and not args[-1]:
            for i in macro.var_comma_patch:
                rep[i] = None
                comma_patch = True

        # Make all other patches.   The order of these matters.  It is assumed that the patch list
        # has been sorted in reverse order of patch location since replacements will cause the
        # size of the replacement sequence to expand from the patch point.
        
        expanded = { }
        #print("***", macro)
        #print(macro.patch)
        for ptype, argnum, i in macro.patch:
            #print([x.value for x in rep])
            # Concatenation.   Argument is left unexpanded
            if ptype == 't':
                rep[i:i+1] = args[argnum]
            # Normal expansion.  Argument is macro expanded first
            elif ptype == 'e':
                #print('*** Function macro arg', rep[i], 'replace with', args[argnum], 'which expands into', self.expand_macros(copy.copy(args[argnum])))
                if argnum not in expanded:
                    expanded[argnum] = self.expand_macros(copy.copy(args[argnum]))
                rep[i:i+1] = expanded[argnum]

        # Get rid of removed comma if necessary
        if comma_patch:
            rep = [_i for _i in rep if _i]
            
        # Do a token concatenation pass, stitching any tokens separated by ## into a single token
        while len(rep) and rep[0].type == self.t_DPOUND:
            del rep[0]
        while len(rep) and rep[-1].type == self.t_DPOUND:
            del rep[-1]
        i = 1
        stitched = False
        while i < len(rep) - 1:
            if rep[i].type == self.t_DPOUND:
                j = i + 1
                while rep[j].type == self.t_DPOUND:
                    j += 1
                rep[i-1] = copy.copy(rep[i-1])
                rep[i-1].type = None
                rep[i-1].value += rep[j].value
                while j >= i:
                    del rep[i]
                    j -= 1
                stitched = True
            else:
                i += 1
        if stitched:
            # Stitched tokens will have unknown type, so figure those out now
            i = 0
            lex = self.lexer.clone()
            while i < len(rep):
                if rep[i].type is None:
                    lex.input(rep[i].value)
                    toks = []
                    while True:
                        tok = lex.token()
                        if not tok:
                            break
                        toks.append(tok)
                    if len(toks) != 1:
                        # Split it once again
                        while len(toks) > 1:
                            rep.insert(i+1, copy.copy(rep[i]))
                            rep[i+1].value = toks[-1].value
                            rep[i+1].type = toks[-1].type
                            toks.pop()
                        rep[i].value = toks[0].value
                        rep[i].type = toks[0].type
                    else:
                        rep[i].type = toks[0].type
                i += 1

        #print rep
        return rep


    # ----------------------------------------------------------------------
    # expand_macros()
    #
    # Given a list of tokens, this function performs macro expansion.
    # ----------------------------------------------------------------------

    def expand_macros(self,tokens,expanding_from=[]):
        """Given a list of tokens, this function performs macro expansion."""
        # Each token needs to track from which macros it has been expanded from to prevent recursion
        for tok in tokens:
            if not hasattr(tok, 'expanded_from'):
                tok.expanded_from = []
        i = 0
        #print("*** EXPAND MACROS in", "".join([t.value for t in tokens]), "expanding_from=", expanding_from)
        #print(tokens)
        #print([(t.value, t.expanded_from) for t in tokens])
        while i < len(tokens):
            t = tokens[i]
            if self.linemacrodepth == 0:
                self.linemacro = t.lineno
            self.linemacrodepth = self.linemacrodepth + 1
            if t.type == self.t_ID:
                if t.value in self.macros and t.value not in t.expanded_from and t.value not in expanding_from:
                    # Yes, we found a macro match
                    m = self.macros[t.value]
                    if m.arglist is None:
                        # A simple macro
                        rep = [copy.copy(_x) for _x in m.value]
                        ex = self.expand_macros(rep, expanding_from + [t.value])
                        #print("\nExpanding macro", m, "\ninto", ex, "\nreplacing", tokens[i:i+1])
                        for e in ex:
                            e.source = t.source
                            e.lineno = t.lineno
                            if not hasattr(e, 'expanded_from'):
                                e.expanded_from = []
                            e.expanded_from.append(t.value)
                        tokens[i:i+1] = ex
                    else:
                        # A macro with arguments
                        j = i + 1
                        while j < len(tokens) and (tokens[j].type in self.t_WS or tokens[j].type in self.t_COMMENT):
                            j += 1
                        # A function like macro without an invocation list is to be ignored
                        if j == len(tokens) or tokens[j].value != '(':
                            i = j
                        else:
                            tokcount,args,positions = self.collect_args(tokens[j:], True)
                            if tokcount == 0:
                                # Unclosed parameter list, just bail out
                                break
                            if (not m.variadic
                                # A no arg or single arg consuming macro is permitted to be expanded with nothing
                                and (args != [[]] or len(m.arglist) > 1)
                                and len(args) !=  len(m.arglist)):
                                self.on_error(t.source,t.lineno,"Macro %s requires %d arguments but was passed %d" % (t.value,len(m.arglist),len(args)))
                                i = j + tokcount
                            elif m.variadic and len(args) < len(m.arglist)-1:
                                if len(m.arglist) > 2:
                                    self.on_error(t.source,t.lineno,"Macro %s must have at least %d arguments" % (t.value, len(m.arglist)-1))
                                else:
                                    self.on_error(t.source,t.lineno,"Macro %s must have at least %d argument" % (t.value, len(m.arglist)-1))
                                i = j + tokcount
                            else:
                                if m.variadic:
                                    if len(args) == len(m.arglist)-1:
                                        args.append([])
                                    else:
                                        args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1]
                                        del args[len(m.arglist):]
                                else:
                                    # If we called a single arg macro with empty, fake extend args
                                    while len(args) < len(m.arglist):
                                        args.append([])
                                        
                                # Get macro replacement text
                                rep = self.macro_expand_args(m,args)
                                ex = self.expand_macros(rep, expanding_from + [t.value])
                                for e in ex:
                                    e.source = t.source
                                    e.lineno = t.lineno
                                    if not hasattr(e, 'expanded_from'):
                                        e.expanded_from = []
                                    e.expanded_from.append(t.value)
                                # A non-conforming extension implemented by the GCC and clang preprocessors
                                # is that an expansion of a macro with arguments where the following token is
                                # an identifier inserts a space between the expansion and the identifier. This
                                # differs from Boost.Wave incidentally (see https://github.com/ned14/pcpp/issues/29)
                                if len(tokens) > j+tokcount and tokens[j+tokcount].type in self.t_ID:
                                    #print("*** token after expansion is", tokens[j+tokcount])
                                    newtok = copy.copy(tokens[j+tokcount])
                                    newtok.type = self.t_SPACE
                                    newtok.value = ' '
                                    ex.append(newtok)
                                #print("\nExpanding macro", m, "\n\ninto", ex, "\n\nreplacing", tokens[i:j+tokcount])
                                tokens[i:j+tokcount] = ex
                    self.linemacrodepth = self.linemacrodepth - 1
                    if self.linemacrodepth == 0:
                        self.linemacro = 0
                    continue
                elif self.expand_linemacro and t.value == '__LINE__':
                    t.type = self.t_INTEGER
                    t.value = self.t_INTEGER_TYPE(self.linemacro)
                elif self.expand_countermacro and t.value == '__COUNTER__':
                    t.type = self.t_INTEGER
                    t.value = self.t_INTEGER_TYPE(self.countermacro)
                    self.countermacro += 1
                
            i += 1
            self.linemacrodepth = self.linemacrodepth - 1
            if self.linemacrodepth == 0:
                self.linemacro = 0
        return tokens

    # ----------------------------------------------------------------------    
    # evalexpr()
    # 
    # Evaluate an expression token sequence for the purposes of evaluating
    # integral expressions.
    # ----------------------------------------------------------------------

    def evalexpr(self,tokens):
        """Evaluate an expression token sequence for the purposes of evaluating
        integral expressions."""
        if not tokens:
            self.on_error('unknown', 0, "Empty expression")
            return (0, None)
        # tokens = tokenize(line)
        # Search for defined macros
        partial_expansion = False
        def replace_defined(tokens):
            i = 0
            while i < len(tokens):
                if tokens[i].type == self.t_ID and tokens[i].value == 'defined':
                    j = i + 1
                    needparen = False
                    result = "0L"
                    while j < len(tokens):
                        if tokens[j].type in self.t_WS:
                            j += 1
                            continue
                        elif tokens[j].type == self.t_ID:
                            if tokens[j].value in self.macros:
                                result = "1L"
                            else:
                                repl = self.on_unknown_macro_in_defined_expr(tokens[j])
                                if repl is None:
                                    partial_expansion = True
                                    result = 'defined('+tokens[j].value+')'
                                else:
                                    result = "1L" if repl else "0L"
                            if not needparen: break
                        elif tokens[j].value == '(':
                            needparen = True
                        elif tokens[j].value == ')':
                            break
                        else:
                            self.on_error(tokens[i].source,tokens[i].lineno,"Malformed defined()")
                        j += 1
                    if result.startswith('defined'):
                        tokens[i].type = self.t_ID
                        tokens[i].value = result
                    else:
                        tokens[i].type = self.t_INTEGER
                        tokens[i].value = self.t_INTEGER_TYPE(result)
                    del tokens[i+1:j+1]
                i += 1
            return tokens
        # Replace any defined(macro) before macro expansion
        tokens = replace_defined(tokens)
        tokens = self.expand_macros(tokens)
        # Replace any defined(macro) after macro expansion
        tokens = replace_defined(tokens)
        if not tokens:
            return (0, None)
        class IndirectToMacroHook(object):
            def __init__(self, p):
                self.__preprocessor = p
                self.partial_expansion = False
            def __contains__(self, key):
                return True
            def __getitem__(self, key):
                if key.startswith('defined('):
                    self.partial_expansion = True
                    return 0
                repl = self.__preprocessor.on_unknown_macro_in_expr(key)
                #print("*** IndirectToMacroHook[", key, "] returns", repl, file = sys.stderr)
                if repl is None:
                    self.partial_expansion = True
                    return key
                return repl
        evalvars = IndirectToMacroHook(self)
        class IndirectToMacroFunctionHook(object):
            def __init__(self, p):
                self.__preprocessor = p
                self.partial_expansion = False
            def __contains__(self, key):
                return True
            def __getitem__(self, key):
                repl = self.__preprocessor.on_unknown_macro_function_in_expr(key)
                #print("*** IndirectToMacroFunctionHook[", key, "] returns", repl, file = sys.stderr)
                if repl is None:
                    self.partial_expansion = True
                    return key
                return repl
        evalfuncts = IndirectToMacroFunctionHook(self)
        try:
            result = self.evaluator(tokens, functions = evalfuncts, identifiers = evalvars).value()
            partial_expansion = partial_expansion or evalvars.partial_expansion or evalfuncts.partial_expansion
        except OutputDirective:
            raise
        except Exception as e:
            partial_expansion = partial_expansion or evalvars.partial_expansion or evalfuncts.partial_expansion
            if not partial_expansion:
                self.on_error(tokens[0].source,tokens[0].lineno,"Could not evaluate expression due to %s (passed to evaluator: '%s')" % (repr(e), ''.join([tok.value for tok in tokens])))
            result = 0
        return (result, tokens) if partial_expansion else (result, None)

    # ----------------------------------------------------------------------
    # parsegen()
    #
    # Parse an input string
    # ----------------------------------------------------------------------
    def parsegen(self,input,source=None,abssource=None):
        """Parse an input string"""
        rewritten_source = source
        if abssource:
            rewritten_source = abssource
            for rewrite in self.rewrite_paths:
                temp = re.sub(rewrite[0], rewrite[1], rewritten_source)
                if temp != abssource:
                    rewritten_source = temp
                    if os.sep != '/':
                        rewritten_source = rewritten_source.replace(os.sep, '/')
                    break

        # Replace trigraph sequences
        t = trigraph(input)
        lines = self.group_lines(t, rewritten_source)

        if not source:
            source = ""
        if not rewritten_source:
            rewritten_source = ""
            
        my_include_times_idx = len(self.include_times)
        self.include_times.append(FileInclusionTime(self.macros['__FILE__'] if '__FILE__' in self.macros else None, source, abssource, self.include_depth))
        self.include_depth += 1
        my_include_time_begin = clock()
        if self.expand_filemacro:
            self.define("__FILE__ \"%s\"" % rewritten_source)

        self.source = abssource
        chunk = []
        enable = True
        iftrigger = False
        ifpassthru = False
        class ifstackentry(object):
            def __init__(self,enable,iftrigger,ifpassthru,startlinetoks):
                self.enable = enable
                self.iftrigger = iftrigger
                self.ifpassthru = ifpassthru
                self.rewritten = False
                self.startlinetoks = startlinetoks
        ifstack = []
        # True until any non-whitespace output or anything with effects happens.
        at_front_of_file = True
        # True if auto pragma once still a possibility for this #include
        auto_pragma_once_possible = self.auto_pragma_once_enabled
        # =(MACRO, 0) means #ifndef MACRO or #if !defined(MACRO) seen, =(MACRO,1) means #define MACRO seen
        include_guard = None
        self.on_potential_include_guard(None)

        for x in lines:
            all_whitespace = True
            skip_auto_pragma_once_possible_check = False
            # Handle comments
            for i,tok in enumerate(x):
                if tok.type in self.t_COMMENT:
                    if not self.on_comment(tok):
                        if tok.type == self.t_COMMENT1:
                            tok.value = ' '
                        elif tok.type == self.t_COMMENT2:
                            tok.value = '\n'
                        tok.type = 'CPP_WS'
            # Skip over whitespace
            for i,tok in enumerate(x):
                if tok.type not in self.t_WS and tok.type not in self.t_COMMENT:
                    all_whitespace = False
                    break
            output_and_expand_line = True
            output_unexpanded_line = False
            if tok.value == '#':
                precedingtoks = [ tok ]
                output_and_expand_line = False
                try:
                    # Preprocessor directive      
                    i += 1
                    while i < len(x) and x[i].type in self.t_WS:
                        precedingtoks.append(x[i])
                        i += 1                    
                    dirtokens = self.tokenstrip(x[i:])
                    if dirtokens:
                        name = dirtokens[0].value
                        args = self.tokenstrip(dirtokens[1:])
                    
                        if self.debugout is not None:
                            print("%d:%d:%d %s:%d #%s %s" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno, dirtokens[0].value, "".join([tok.value for tok in args])), file = self.debugout)
                            #print(ifstack)

                        handling = self.on_directive_handle(dirtokens[0],args,ifpassthru,precedingtoks)
                        assert handling == True or handling == None
                    else:
                        name = ""
                        args = []
                        raise OutputDirective(Action.IgnoreAndRemove)
                        
                    if name == 'define':
                        at_front_of_file = False
                        if enable:
                            for tok in self.expand_macros(chunk):
                                yield tok
                            chunk = []
                            if include_guard and include_guard[1] == 0:
                                if include_guard[0] == args[0].value and len(args) == 1:
                                    include_guard = (args[0].value, 1)
                                    # If ifpassthru is only turned on due to this include guard, turn it off
                                    if ifpassthru and not ifstack[-1].ifpassthru:
                                        ifpassthru = False
                            self.define(args)
                            if self.debugout is not None:
                                print("%d:%d:%d %s:%d      %s" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno, repr(self.macros[args[0].value])), file = self.debugout)
                            if handling is None:
                                for tok in x:
                                    yield tok
                    elif name == 'include':
                        if enable:
                            for tok in self.expand_macros(chunk):
                                yield tok
                            chunk = []
                            oldfile = self.macros['__FILE__'] if '__FILE__' in self.macros else None
                            if args and args[0].value != '<' and args[0].type != self.t_STRING:
                                args = self.tokenstrip(self.expand_macros(args))
                            #print('***', ''.join([x.value for x in args]), file = sys.stderr)
                            if self.passthru_includes is not None and self.passthru_includes.match(''.join([x.value for x in args])):
                                for tok in precedingtoks:
                                    yield tok
                                for tok in dirtokens:
                                    yield tok
                                for tok in self.include(args):
                                    pass
                            else:
                                for tok in self.include(args):
                                    yield tok
                            if oldfile is not None:
                                self.macros['__FILE__'] = oldfile
                            self.source = abssource
                    elif name == 'undef':
                        at_front_of_file = False
                        if enable:
                            for tok in self.expand_macros(chunk):
                                yield tok
                            chunk = []
                            self.undef(args)
                            if handling is None:
                                for tok in x:
                                    yield tok
                    elif name == 'ifdef':
                        at_front_of_file = False
                        ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x))
                        if enable:
                            ifpassthru = False
                            if not args[0].value in self.macros:
                                res = self.on_unknown_macro_in_defined_expr(args[0])
                                if res is None:
                                    ifpassthru = True
                                    ifstack[-1].rewritten = True
                                    raise OutputDirective(Action.IgnoreAndPassThrough)
                                elif res is True:
                                    iftrigger = True
                                else:
                                    enable = False
                                    iftrigger = False
                            else:
                                iftrigger = True
                    elif name == 'ifndef':
                        if not ifstack and at_front_of_file:
                            self.on_potential_include_guard(args[0].value)
                            include_guard = (args[0].value, 0)
                        at_front_of_file = False
                        ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x))
                        if enable:
                            ifpassthru = False
                            if args[0].value in self.macros:
                                enable = False
                                iftrigger = False
                            else:
                                res = self.on_unknown_macro_in_defined_expr(args[0])
                                if res is None:
                                    ifpassthru = True
                                    ifstack[-1].rewritten = True
                                    raise OutputDirective(Action.IgnoreAndPassThrough)
                                elif res is True:
                                    enable = False
                                    iftrigger = False
                                else:
                                    iftrigger = True
                    elif name == 'if':
                        if not ifstack and at_front_of_file:
                            if args[0].value == '!' and args[1].value == 'defined':
                                n = 2
                                if args[n].value == '(': n += 1
                                self.on_potential_include_guard(args[n].value)
                                include_guard = (args[n].value, 0)
                        at_front_of_file = False
                        ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x))
                        if enable:
                            iftrigger = False
                            ifpassthru = False
                            result, rewritten = self.evalexpr(args)
                            if rewritten is not None:
                                x = x[:i+2] + rewritten + [x[-1]]
                                x[i+1] = copy.copy(x[i+1])
                                x[i+1].type = self.t_SPACE
                                x[i+1].value = ' '
                                ifpassthru = True
                                ifstack[-1].rewritten = True
                                raise OutputDirective(Action.IgnoreAndPassThrough)
                            if not result:
                                enable = False
                            else:
                                iftrigger = True
                    elif name == 'elif':
                        at_front_of_file = False
                        if ifstack:
                            if ifstack[-1].enable:     # We only pay attention if outer "if" allows this
                                if enable and not ifpassthru:         # If already true, we flip enable False
                                    enable = False
                                elif not iftrigger:   # If False, but not triggered yet, we'll check expression
                                    result, rewritten = self.evalexpr(args)
                                    if rewritten is not None:
                                        enable = True
                                        if not ifpassthru:
                                            # This is a passthru #elif after a False #if, so convert to an #if
                                            x[i].value = 'if'
                                        x = x[:i+2] + rewritten + [x[-1]]
                                        x[i+1] = copy.copy(x[i+1])
                                        x[i+1].type = self.t_SPACE
                                        x[i+1].value = ' '
                                        ifpassthru = True
                                        ifstack[-1].rewritten = True
                                        raise OutputDirective(Action.IgnoreAndPassThrough)
                                    if ifpassthru:
                                        # If this elif can only ever be true, simulate that
                                        if result:
                                            newtok = copy.copy(x[i+3])
                                            newtok.type = self.t_INTEGER
                                            newtok.value = self.t_INTEGER_TYPE(result)
                                            x = x[:i+2] + [newtok] + [x[-1]]
                                            raise OutputDirective(Action.IgnoreAndPassThrough)
                                        # Otherwise elide
                                        enable = False
                                    elif result:
                                        enable  = True
                                        iftrigger = True
                        else:
                            self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #elif")
                            
                    elif name == 'else':
                        at_front_of_file = False
                        if ifstack:
                            if ifstack[-1].enable:
                                if ifpassthru:
                                    enable = True
                                    raise OutputDirective(Action.IgnoreAndPassThrough)
                                if enable:
                                    enable = False
                                elif not iftrigger:
                                    enable = True
                                    iftrigger = True
                        else:
                            self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #else")

                    elif name == 'endif':
                        at_front_of_file = False
                        if ifstack:
                            oldifstackentry = ifstack.pop()
                            enable = oldifstackentry.enable
                            iftrigger = oldifstackentry.iftrigger
                            ifpassthru = oldifstackentry.ifpassthru
                            if self.debugout is not None:
                                print("%d:%d:%d %s:%d      (%s:%d %s)" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno,
                                    oldifstackentry.startlinetoks[0].source, oldifstackentry.startlinetoks[0].lineno, "".join([n.value for n in oldifstackentry.startlinetoks])), file = self.debugout)
                            skip_auto_pragma_once_possible_check = True
                            if oldifstackentry.rewritten:
                                raise OutputDirective(Action.IgnoreAndPassThrough)
                        else:
                            self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #endif")
                    elif name == 'pragma' and args[0].value == 'once':
                        if enable:
                            self.include_once[self.source] = None
                    elif enable:
                        # Unknown preprocessor directive
                        output_unexpanded_line = (self.on_directive_unknown(dirtokens[0], args, ifpassthru, precedingtoks) is None)

                except OutputDirective as e:
                    if e.action == Action.IgnoreAndPassThrough:
                        output_unexpanded_line = True
                    elif e.action == Action.IgnoreAndRemove:
                        pass
                    else:
                        assert False

            # If there is ever any non-whitespace output outside an include guard, auto pragma once is not possible
            if not skip_auto_pragma_once_possible_check and auto_pragma_once_possible and not ifstack and not all_whitespace:
                auto_pragma_once_possible = False
                if self.debugout is not None:
                    print("%d:%d:%d %s:%d Determined that #include \"%s\" is not entirely wrapped in an include guard macro, disabling auto-applying #pragma once" % (enable, iftrigger, ifpassthru, x[0].source, x[0].lineno, self.source), file = self.debugout)
                
            if output_and_expand_line or output_unexpanded_line:
                if not all_whitespace:
                    at_front_of_file = False

                # Normal text
                if enable:
                    if output_and_expand_line:
                        chunk.extend(x)
                    elif output_unexpanded_line:
                        for tok in self.expand_macros(chunk):
                            yield tok
                        chunk = []
                        for tok in x:
                            yield tok
                else:
                    # Need to extend with the same number of blank lines
                    i = 0
                    while i < len(x):
                        if x[i].type not in self.t_WS:
                            del x[i]
                        else:
                            i += 1
                    chunk.extend(x)

        for tok in self.expand_macros(chunk):
            yield tok
        chunk = []
        for i in ifstack:
            self.on_error(i.startlinetoks[0].source, i.startlinetoks[0].lineno, "Unterminated " + "".join([n.value for n in i.startlinetoks]))
        if auto_pragma_once_possible and include_guard and include_guard[1] == 1:
            if self.debugout is not None:
                print("%d:%d:%d %s:%d Determined that #include \"%s\" is entirely wrapped in an include guard macro called %s, auto-applying #pragma once" % (enable, iftrigger, ifpassthru, self.source, 0, self.source, include_guard[0]), file = self.debugout)
            self.include_once[self.source] = include_guard[0]
        elif self.auto_pragma_once_enabled and self.source not in self.include_once:
            if self.debugout is not None:
                print("%d:%d:%d %s:%d Did not auto apply #pragma once to this file due to auto_pragma_once_possible=%d, include_guard=%s" % (enable, iftrigger, ifpassthru, self.source, 0, auto_pragma_once_possible, repr(include_guard)), file = self.debugout)
        my_include_time_end = clock()
        self.include_times[my_include_times_idx].elapsed = my_include_time_end - my_include_time_begin
        self.include_depth -= 1

    # ----------------------------------------------------------------------
    # include()
    #
    # Implementation of file-inclusion
    # ----------------------------------------------------------------------

    def include(self,tokens):
        """Implementation of file-inclusion"""
        # Try to extract the filename and then process an include file
        if not tokens:
            return
        if tokens:
            if tokens[0].value != '<' and tokens[0].type != self.t_STRING:
                tokens = self.tokenstrip(self.expand_macros(tokens))

            is_system_include = False
            if tokens[0].value == '<':
                is_system_include = True
                # Include <...>
                i = 1
                while i < len(tokens):
                    if tokens[i].value == '>':
                        break
                    i += 1
                else:
                    self.on_error(tokens[0].source,tokens[0].lineno,"Malformed #include <...>")
                    return
                filename = "".join([x.value for x in tokens[1:i]])
                # Search only formally specified paths
                path = self.path
            elif tokens[0].type == self.t_STRING:
                filename = tokens[0].value[1:-1]
                # Search from each nested include file, as well as formally specified paths
                path = self.temp_path + self.path
            else:
                p = self.on_include_not_found(True,False,self.temp_path[0] if self.temp_path else '',tokens[0].value)
                assert p is None
                return
        if not path:
            path = ['']
        while True:
            #print path
            for p in path:
                iname = os.path.join(p,filename)
                fulliname = os.path.abspath(iname)
                if fulliname in self.include_once:
                    if self.debugout is not None:
                        print("x:x:x x:x #include \"%s\" skipped as already seen" % (fulliname), file = self.debugout)
                    return
                try:
                    ih = self.on_file_open(is_system_include,fulliname)
                    data = ih.read()
                    ih.close()
                    dname = os.path.dirname(fulliname)
                    if dname:
                        self.temp_path.insert(0,dname)
                    for tok in self.parsegen(data,filename,fulliname):
                        yield tok
                    if dname:
                        del self.temp_path[0]
                    return
                except IOError:
                    pass
            else:
                p = self.on_include_not_found(False,is_system_include,self.temp_path[0] if self.temp_path else '',filename)
                assert p is not None
                path.append(p)

    # ----------------------------------------------------------------------
    # define()
    #
    # Define a new macro
    # ----------------------------------------------------------------------

    def define(self,tokens):
        """Define a new macro"""
        if isinstance(tokens,STRING_TYPES):
            tokens = self.tokenize(tokens)
        else:
            tokens = [copy.copy(tok) for tok in tokens]
        def add_macro(self, name, macro):
            macro.source = name.source
            macro.lineno = name.lineno
            self.macros[name.value] = macro

        linetok = tokens
        try:
            name = linetok[0]
            if len(linetok) > 1:
                mtype = linetok[1]
            else:
                mtype = None
            if not mtype:
                m = Macro(name.value,[])
                add_macro(self, name, m)
            elif mtype.type in self.t_WS:
                # A normal macro
                m = Macro(name.value,self.tokenstrip(linetok[2:]))
                add_macro(self, name, m)
            elif mtype.value == '(':
                # A macro with arguments
                tokcount, args, positions = self.collect_args(linetok[1:])
                variadic = False
                for a in args:
                    if variadic:
                        self.on_error(name.source,name.lineno,"No more arguments may follow a variadic argument")
                        break
                    astr = "".join([str(_i.value) for _i in a])
                    if astr == "...":
                        variadic = True
                        a[0].type = self.t_ID
                        a[0].value = '__VA_ARGS__'
                        variadic = True
                        del a[1:]
                        continue
                    elif astr[-3:] == "..." and a[0].type == self.t_ID:
                        variadic = True
                        del a[1:]
                        # If, for some reason, "." is part of the identifier, strip off the name for the purposes
                        # of macro expansion
                        if a[0].value[-3:] == '...':
                            a[0].value = a[0].value[:-3]
                        continue
                    # Empty arguments are permitted
                    if len(a) == 0 and len(args) == 1:
                        continue
                    if len(a) > 1 or a[0].type != self.t_ID:
                        self.on_error(a[0].source,a[0].lineno,"Invalid macro argument")
                        break
                else:
                    mvalue = self.tokenstrip(linetok[1+tokcount:])
                    i = 0
                    while i < len(mvalue):
                        if i+1 < len(mvalue):
                            if mvalue[i].type in self.t_WS and mvalue[i+1].value == '##':
                                del mvalue[i]
                                continue
                            elif mvalue[i].value == '##' and mvalue[i+1].type in self.t_WS:
                                del mvalue[i+1]
                        i += 1
                    m = Macro(name.value,mvalue,[x[0].value for x in args] if args != [[]] else [],variadic)
                    self.macro_prescan(m)
                    add_macro(self, name, m)
            else:
                self.on_error(name.source,name.lineno,"Bad macro definition")
        #except LookupError:
        #    print("Bad macro definition")
        except:
            raise

    # ----------------------------------------------------------------------
    # undef()
    #
    # Undefine a macro
    # ----------------------------------------------------------------------

    def undef(self,tokens):
        """Undefine a macro"""
        if isinstance(tokens,STRING_TYPES):
            tokens = self.tokenize(tokens)
        id = tokens[0].value
        try:
            del self.macros[id]
        except LookupError:
            pass

    # ----------------------------------------------------------------------
    # parse()
    #
    # Parse input text.
    # ----------------------------------------------------------------------
    def parse(self,input,source=None,ignore={}):
        """Parse input text."""
        if isinstance(input, FILE_TYPES):
            if source is None:
                source = input.name
            input = input.read()
        self.ignore = ignore
        self.parser = self.parsegen(input,source,os.path.abspath(source) if source else None)
        if source is not None:
            dname = os.path.dirname(source)
            self.temp_path.insert(0,dname)
        
    # ----------------------------------------------------------------------
    # token()
    #
    # Method to return individual tokens
    # ----------------------------------------------------------------------
    def token(self):
        """Method to return individual tokens"""
        try:
            while True:
                tok = next(self.parser)
                if tok.type not in self.ignore:
                    return tok
        except StopIteration:
            self.parser = None
            return None
            
    def write(self, oh=sys.stdout):
        """Calls token() repeatedly, expanding tokens to their text and writing to the file like stream oh"""
        lastlineno = 0
        lastsource = None
        done = False
        blanklines = 0
        while not done:
            emitlinedirective = False
            toks = []
            all_ws = True
            # Accumulate a line
            while not done:
                tok = self.token()
                if not tok:
                    done = True
                    break
                toks.append(tok)
                if tok.value and tok.value[0] == '\n':
                    break
                if tok.type not in self.t_WS:
                    all_ws = False
            if not toks:
                break
            if all_ws:
                # Remove preceding whitespace so it becomes just a LF
                if len(toks) > 1:
                    tok = toks[-1]
                    toks = [ tok ]
                blanklines += toks[0].value.count('\n')
                continue
            # Filter out line continuations, collapsing before and after if needs be
            for n in xrange(len(toks)-1, -1, -1):
                if toks[n].type in self.t_LINECONT:
                    if n > 0 and n < len(toks) - 1 and toks[n-1].type in self.t_WS and toks[n+1].type in self.t_WS:
                        toks[n-1].value = toks[n-1].value[0]
                        del toks[n:n+2]
                    else:
                        del toks[n]
            # The line in toks is not all whitespace
            emitlinedirective = (blanklines > 6) and self.line_directive is not None
            if hasattr(toks[0], 'source'):
                if lastsource is None:
                    if toks[0].source is not None:
                        emitlinedirective = True
                    lastsource = toks[0].source
                elif lastsource != toks[0].source:
                    emitlinedirective = True
                    lastsource = toks[0].source
            # Replace consecutive whitespace in output with a single space except at any indent
            first_ws = None
            #print(toks)
            for n in xrange(len(toks)-1, -1, -1):
                tok = toks[n]
                if first_ws is None:
                    if tok.type in self.t_SPACE or len(tok.value) == 0:
                        first_ws = n
                else:
                    if tok.type not in self.t_SPACE and len(tok.value) > 0:
                        m = n + 1
                        while m != first_ws:
                            del toks[m]
                            first_ws -= 1
                        first_ws = None
                        if self.compress > 0:
                            # Collapse a token of many whitespace into single
                            if toks[m].value and toks[m].value[0] == ' ':
                                toks[m].value = ' '
            if not self.compress > 1 and not emitlinedirective:
                newlinesneeded = toks[0].lineno - lastlineno - 1
                if newlinesneeded > 6 and self.line_directive is not None:
                    emitlinedirective = True
                else:
                    while newlinesneeded > 0:
                        oh.write('\n')
                        newlinesneeded -= 1
            lastlineno = toks[0].lineno
            # Account for those newlines in a multiline comment
            if emitlinedirective and self.line_directive is not None:
                oh.write(self.line_directive + ' ' + str(lastlineno) + ('' if lastsource is None else (' "' + lastsource + '"' )) + '\n')
            for tok in toks:
                if tok.type == self.t_COMMENT1:
                    lastlineno += tok.value.count('\n')
            blanklines = 0
            #print toks[0].lineno, 
            for tok in toks:
                #print tok.value,
                oh.write(tok.value)

Methods

def add_path(self, path)

Adds a search path to the preprocessor.

Source code
def add_path(self,path):
    """Adds a search path to the preprocessor. """
    self.path.append(path)
    # If the search path being added is relative, or has a common ancestor to the
    # current working directory, add a rewrite to relativise includes from this
    # search path
    relpath = None
    try:
        relpath = os.path.relpath(path)
    except: pass
    if relpath is not None:
        self.rewrite_paths += [(re.escape(os.path.abspath(path) + os.sep) + '(.*)', os.path.join(relpath, '\\1'))]
def collect_args(self, tokenlist, ignore_errors=False)

Collects comma separated arguments from a list of tokens. The arguments must be enclosed in parenthesis. Returns a tuple (tokencount,args,positions) where tokencount is the number of tokens consumed, args is a list of arguments, and positions is a list of integers containing the starting index of each argument. Each argument is represented by a list of tokens.

When collecting arguments, leading and trailing whitespace is removed from each argument.

This function properly handles nested parenthesis and commas—these do not define new arguments.

Source code
def collect_args(self,tokenlist,ignore_errors=False):
    """Collects comma separated arguments from a list of tokens.   The arguments
    must be enclosed in parenthesis.  Returns a tuple (tokencount,args,positions)
    where tokencount is the number of tokens consumed, args is a list of arguments,
    and positions is a list of integers containing the starting index of each
    argument.  Each argument is represented by a list of tokens.
    
    When collecting arguments, leading and trailing whitespace is removed
    from each argument.  
    
    This function properly handles nested parenthesis and commas---these do not
    define new arguments."""
    args = []
    positions = []
    current_arg = []
    nesting = 1
    tokenlen = len(tokenlist)

    # Search for the opening '('.
    i = 0
    while (i < tokenlen) and (tokenlist[i].type in self.t_WS):
        i += 1

    if (i < tokenlen) and (tokenlist[i].value == '('):
        positions.append(i+1)
    else:
        if not ignore_errors:
            self.on_error(tokenlist[0].source,tokenlist[0].lineno,"Missing '(' in macro arguments")
        return 0, [], []

    i += 1

    while i < tokenlen:
        t = tokenlist[i]
        if t.value == '(':
            current_arg.append(t)
            nesting += 1
        elif t.value == ')':
            nesting -= 1
            if nesting == 0:
                args.append(self.tokenstrip(current_arg))
                positions.append(i)
                return i+1,args,positions
            current_arg.append(t)
        elif t.value == ',' and nesting == 1:
            args.append(self.tokenstrip(current_arg))
            positions.append(i+1)
            current_arg = []
        else:
            current_arg.append(t)
        i += 1

    # Missing end argument
    if not ignore_errors:
        self.on_error(tokenlist[-1].source,tokenlist[-1].lineno,"Missing ')' in macro arguments")
    return 0, [],[]
def define(self, tokens)

Define a new macro

Source code
def define(self,tokens):
    """Define a new macro"""
    if isinstance(tokens,STRING_TYPES):
        tokens = self.tokenize(tokens)
    else:
        tokens = [copy.copy(tok) for tok in tokens]
    def add_macro(self, name, macro):
        macro.source = name.source
        macro.lineno = name.lineno
        self.macros[name.value] = macro

    linetok = tokens
    try:
        name = linetok[0]
        if len(linetok) > 1:
            mtype = linetok[1]
        else:
            mtype = None
        if not mtype:
            m = Macro(name.value,[])
            add_macro(self, name, m)
        elif mtype.type in self.t_WS:
            # A normal macro
            m = Macro(name.value,self.tokenstrip(linetok[2:]))
            add_macro(self, name, m)
        elif mtype.value == '(':
            # A macro with arguments
            tokcount, args, positions = self.collect_args(linetok[1:])
            variadic = False
            for a in args:
                if variadic:
                    self.on_error(name.source,name.lineno,"No more arguments may follow a variadic argument")
                    break
                astr = "".join([str(_i.value) for _i in a])
                if astr == "...":
                    variadic = True
                    a[0].type = self.t_ID
                    a[0].value = '__VA_ARGS__'
                    variadic = True
                    del a[1:]
                    continue
                elif astr[-3:] == "..." and a[0].type == self.t_ID:
                    variadic = True
                    del a[1:]
                    # If, for some reason, "." is part of the identifier, strip off the name for the purposes
                    # of macro expansion
                    if a[0].value[-3:] == '...':
                        a[0].value = a[0].value[:-3]
                    continue
                # Empty arguments are permitted
                if len(a) == 0 and len(args) == 1:
                    continue
                if len(a) > 1 or a[0].type != self.t_ID:
                    self.on_error(a[0].source,a[0].lineno,"Invalid macro argument")
                    break
            else:
                mvalue = self.tokenstrip(linetok[1+tokcount:])
                i = 0
                while i < len(mvalue):
                    if i+1 < len(mvalue):
                        if mvalue[i].type in self.t_WS and mvalue[i+1].value == '##':
                            del mvalue[i]
                            continue
                        elif mvalue[i].value == '##' and mvalue[i+1].type in self.t_WS:
                            del mvalue[i+1]
                    i += 1
                m = Macro(name.value,mvalue,[x[0].value for x in args] if args != [[]] else [],variadic)
                self.macro_prescan(m)
                add_macro(self, name, m)
        else:
            self.on_error(name.source,name.lineno,"Bad macro definition")
    #except LookupError:
    #    print("Bad macro definition")
    except:
        raise
def evalexpr(self, tokens)

Evaluate an expression token sequence for the purposes of evaluating integral expressions.

Source code
def evalexpr(self,tokens):
    """Evaluate an expression token sequence for the purposes of evaluating
    integral expressions."""
    if not tokens:
        self.on_error('unknown', 0, "Empty expression")
        return (0, None)
    # tokens = tokenize(line)
    # Search for defined macros
    partial_expansion = False
    def replace_defined(tokens):
        i = 0
        while i < len(tokens):
            if tokens[i].type == self.t_ID and tokens[i].value == 'defined':
                j = i + 1
                needparen = False
                result = "0L"
                while j < len(tokens):
                    if tokens[j].type in self.t_WS:
                        j += 1
                        continue
                    elif tokens[j].type == self.t_ID:
                        if tokens[j].value in self.macros:
                            result = "1L"
                        else:
                            repl = self.on_unknown_macro_in_defined_expr(tokens[j])
                            if repl is None:
                                partial_expansion = True
                                result = 'defined('+tokens[j].value+')'
                            else:
                                result = "1L" if repl else "0L"
                        if not needparen: break
                    elif tokens[j].value == '(':
                        needparen = True
                    elif tokens[j].value == ')':
                        break
                    else:
                        self.on_error(tokens[i].source,tokens[i].lineno,"Malformed defined()")
                    j += 1
                if result.startswith('defined'):
                    tokens[i].type = self.t_ID
                    tokens[i].value = result
                else:
                    tokens[i].type = self.t_INTEGER
                    tokens[i].value = self.t_INTEGER_TYPE(result)
                del tokens[i+1:j+1]
            i += 1
        return tokens
    # Replace any defined(macro) before macro expansion
    tokens = replace_defined(tokens)
    tokens = self.expand_macros(tokens)
    # Replace any defined(macro) after macro expansion
    tokens = replace_defined(tokens)
    if not tokens:
        return (0, None)
    class IndirectToMacroHook(object):
        def __init__(self, p):
            self.__preprocessor = p
            self.partial_expansion = False
        def __contains__(self, key):
            return True
        def __getitem__(self, key):
            if key.startswith('defined('):
                self.partial_expansion = True
                return 0
            repl = self.__preprocessor.on_unknown_macro_in_expr(key)
            #print("*** IndirectToMacroHook[", key, "] returns", repl, file = sys.stderr)
            if repl is None:
                self.partial_expansion = True
                return key
            return repl
    evalvars = IndirectToMacroHook(self)
    class IndirectToMacroFunctionHook(object):
        def __init__(self, p):
            self.__preprocessor = p
            self.partial_expansion = False
        def __contains__(self, key):
            return True
        def __getitem__(self, key):
            repl = self.__preprocessor.on_unknown_macro_function_in_expr(key)
            #print("*** IndirectToMacroFunctionHook[", key, "] returns", repl, file = sys.stderr)
            if repl is None:
                self.partial_expansion = True
                return key
            return repl
    evalfuncts = IndirectToMacroFunctionHook(self)
    try:
        result = self.evaluator(tokens, functions = evalfuncts, identifiers = evalvars).value()
        partial_expansion = partial_expansion or evalvars.partial_expansion or evalfuncts.partial_expansion
    except OutputDirective:
        raise
    except Exception as e:
        partial_expansion = partial_expansion or evalvars.partial_expansion or evalfuncts.partial_expansion
        if not partial_expansion:
            self.on_error(tokens[0].source,tokens[0].lineno,"Could not evaluate expression due to %s (passed to evaluator: '%s')" % (repr(e), ''.join([tok.value for tok in tokens])))
        result = 0
    return (result, tokens) if partial_expansion else (result, None)
def expand_macros(self, tokens, expanding_from=[])

Given a list of tokens, this function performs macro expansion.

Source code
def expand_macros(self,tokens,expanding_from=[]):
    """Given a list of tokens, this function performs macro expansion."""
    # Each token needs to track from which macros it has been expanded from to prevent recursion
    for tok in tokens:
        if not hasattr(tok, 'expanded_from'):
            tok.expanded_from = []
    i = 0
    #print("*** EXPAND MACROS in", "".join([t.value for t in tokens]), "expanding_from=", expanding_from)
    #print(tokens)
    #print([(t.value, t.expanded_from) for t in tokens])
    while i < len(tokens):
        t = tokens[i]
        if self.linemacrodepth == 0:
            self.linemacro = t.lineno
        self.linemacrodepth = self.linemacrodepth + 1
        if t.type == self.t_ID:
            if t.value in self.macros and t.value not in t.expanded_from and t.value not in expanding_from:
                # Yes, we found a macro match
                m = self.macros[t.value]
                if m.arglist is None:
                    # A simple macro
                    rep = [copy.copy(_x) for _x in m.value]
                    ex = self.expand_macros(rep, expanding_from + [t.value])
                    #print("\nExpanding macro", m, "\ninto", ex, "\nreplacing", tokens[i:i+1])
                    for e in ex:
                        e.source = t.source
                        e.lineno = t.lineno
                        if not hasattr(e, 'expanded_from'):
                            e.expanded_from = []
                        e.expanded_from.append(t.value)
                    tokens[i:i+1] = ex
                else:
                    # A macro with arguments
                    j = i + 1
                    while j < len(tokens) and (tokens[j].type in self.t_WS or tokens[j].type in self.t_COMMENT):
                        j += 1
                    # A function like macro without an invocation list is to be ignored
                    if j == len(tokens) or tokens[j].value != '(':
                        i = j
                    else:
                        tokcount,args,positions = self.collect_args(tokens[j:], True)
                        if tokcount == 0:
                            # Unclosed parameter list, just bail out
                            break
                        if (not m.variadic
                            # A no arg or single arg consuming macro is permitted to be expanded with nothing
                            and (args != [[]] or len(m.arglist) > 1)
                            and len(args) !=  len(m.arglist)):
                            self.on_error(t.source,t.lineno,"Macro %s requires %d arguments but was passed %d" % (t.value,len(m.arglist),len(args)))
                            i = j + tokcount
                        elif m.variadic and len(args) < len(m.arglist)-1:
                            if len(m.arglist) > 2:
                                self.on_error(t.source,t.lineno,"Macro %s must have at least %d arguments" % (t.value, len(m.arglist)-1))
                            else:
                                self.on_error(t.source,t.lineno,"Macro %s must have at least %d argument" % (t.value, len(m.arglist)-1))
                            i = j + tokcount
                        else:
                            if m.variadic:
                                if len(args) == len(m.arglist)-1:
                                    args.append([])
                                else:
                                    args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1]
                                    del args[len(m.arglist):]
                            else:
                                # If we called a single arg macro with empty, fake extend args
                                while len(args) < len(m.arglist):
                                    args.append([])
                                    
                            # Get macro replacement text
                            rep = self.macro_expand_args(m,args)
                            ex = self.expand_macros(rep, expanding_from + [t.value])
                            for e in ex:
                                e.source = t.source
                                e.lineno = t.lineno
                                if not hasattr(e, 'expanded_from'):
                                    e.expanded_from = []
                                e.expanded_from.append(t.value)
                            # A non-conforming extension implemented by the GCC and clang preprocessors
                            # is that an expansion of a macro with arguments where the following token is
                            # an identifier inserts a space between the expansion and the identifier. This
                            # differs from Boost.Wave incidentally (see https://github.com/ned14/pcpp/issues/29)
                            if len(tokens) > j+tokcount and tokens[j+tokcount].type in self.t_ID:
                                #print("*** token after expansion is", tokens[j+tokcount])
                                newtok = copy.copy(tokens[j+tokcount])
                                newtok.type = self.t_SPACE
                                newtok.value = ' '
                                ex.append(newtok)
                            #print("\nExpanding macro", m, "\n\ninto", ex, "\n\nreplacing", tokens[i:j+tokcount])
                            tokens[i:j+tokcount] = ex
                self.linemacrodepth = self.linemacrodepth - 1
                if self.linemacrodepth == 0:
                    self.linemacro = 0
                continue
            elif self.expand_linemacro and t.value == '__LINE__':
                t.type = self.t_INTEGER
                t.value = self.t_INTEGER_TYPE(self.linemacro)
            elif self.expand_countermacro and t.value == '__COUNTER__':
                t.type = self.t_INTEGER
                t.value = self.t_INTEGER_TYPE(self.countermacro)
                self.countermacro += 1
            
        i += 1
        self.linemacrodepth = self.linemacrodepth - 1
        if self.linemacrodepth == 0:
            self.linemacro = 0
    return tokens
def group_lines(self, input, abssource)

Given an input string, this function splits it into lines. Trailing whitespace is removed. This function forms the lowest level of the preprocessor—grouping text into a line-by-line format.

Source code
def group_lines(self,input,abssource):
    r"""Given an input string, this function splits it into lines.  Trailing whitespace
    is removed. This function forms the lowest level of the preprocessor---grouping
    text into a line-by-line format.
    """
    lex = self.lexer.clone()
    lines = [x.rstrip() for x in input.splitlines()]

    input = "\n".join(lines)
    lex.input(input)
    lex.lineno = 1

    current_line = []
    while True:
        tok = lex.token()
        if not tok:
            break
        tok.source = abssource
        current_line.append(tok)
        if tok.type in self.t_WS and tok.value == '\n':
            yield current_line
            current_line = []

    if current_line:
        nltok = copy.copy(current_line[-1])
        nltok.type = self.t_NEWLINE
        nltok.value = '\n'
        current_line.append(nltok)
        yield current_line
def include(self, tokens)

Implementation of file-inclusion

Source code
def include(self,tokens):
    """Implementation of file-inclusion"""
    # Try to extract the filename and then process an include file
    if not tokens:
        return
    if tokens:
        if tokens[0].value != '<' and tokens[0].type != self.t_STRING:
            tokens = self.tokenstrip(self.expand_macros(tokens))

        is_system_include = False
        if tokens[0].value == '<':
            is_system_include = True
            # Include <...>
            i = 1
            while i < len(tokens):
                if tokens[i].value == '>':
                    break
                i += 1
            else:
                self.on_error(tokens[0].source,tokens[0].lineno,"Malformed #include <...>")
                return
            filename = "".join([x.value for x in tokens[1:i]])
            # Search only formally specified paths
            path = self.path
        elif tokens[0].type == self.t_STRING:
            filename = tokens[0].value[1:-1]
            # Search from each nested include file, as well as formally specified paths
            path = self.temp_path + self.path
        else:
            p = self.on_include_not_found(True,False,self.temp_path[0] if self.temp_path else '',tokens[0].value)
            assert p is None
            return
    if not path:
        path = ['']
    while True:
        #print path
        for p in path:
            iname = os.path.join(p,filename)
            fulliname = os.path.abspath(iname)
            if fulliname in self.include_once:
                if self.debugout is not None:
                    print("x:x:x x:x #include \"%s\" skipped as already seen" % (fulliname), file = self.debugout)
                return
            try:
                ih = self.on_file_open(is_system_include,fulliname)
                data = ih.read()
                ih.close()
                dname = os.path.dirname(fulliname)
                if dname:
                    self.temp_path.insert(0,dname)
                for tok in self.parsegen(data,filename,fulliname):
                    yield tok
                if dname:
                    del self.temp_path[0]
                return
            except IOError:
                pass
        else:
            p = self.on_include_not_found(False,is_system_include,self.temp_path[0] if self.temp_path else '',filename)
            assert p is not None
            path.append(p)
def macro_expand_args(self, macro, args)

Given a Macro and list of arguments (each a token list), this method returns an expanded version of a macro. The return value is a token sequence representing the replacement macro tokens

Source code
def macro_expand_args(self,macro,args):
    """Given a Macro and list of arguments (each a token list), this method
    returns an expanded version of a macro.  The return value is a token sequence
    representing the replacement macro tokens"""
    # Make a copy of the macro token sequence
    rep = [copy.copy(_x) for _x in macro.value]

    # Make string expansion patches.  These do not alter the length of the replacement sequence
    str_expansion = {}
    for argnum, i in macro.str_patch:
        if argnum not in str_expansion:
            # Strip all non-space whitespace before stringization
            tokens = copy.copy(args[argnum])
            for j in xrange(len(tokens)):
                if tokens[j].type in self.t_WS and tokens[j].type != self.t_LINECONT:
                    tokens[j].value = ' '
            # Collapse all multiple whitespace too
            j = 0
            while j < len(tokens) - 1:
                if tokens[j].type in self.t_WS and tokens[j+1].type in self.t_WS:
                    del tokens[j+1]
                else:
                    j += 1
            str = "".join([x.value for x in tokens])
            str = str.replace("\\","\\\\").replace('"', '\\"')
            str_expansion[argnum] = '"' + str + '"'
        rep[i] = copy.copy(rep[i])
        rep[i].value = str_expansion[argnum]

    # Make the variadic macro comma patch.  If the variadic macro argument is empty, we get rid
    comma_patch = False
    if macro.variadic and not args[-1]:
        for i in macro.var_comma_patch:
            rep[i] = None
            comma_patch = True

    # Make all other patches.   The order of these matters.  It is assumed that the patch list
    # has been sorted in reverse order of patch location since replacements will cause the
    # size of the replacement sequence to expand from the patch point.
    
    expanded = { }
    #print("***", macro)
    #print(macro.patch)
    for ptype, argnum, i in macro.patch:
        #print([x.value for x in rep])
        # Concatenation.   Argument is left unexpanded
        if ptype == 't':
            rep[i:i+1] = args[argnum]
        # Normal expansion.  Argument is macro expanded first
        elif ptype == 'e':
            #print('*** Function macro arg', rep[i], 'replace with', args[argnum], 'which expands into', self.expand_macros(copy.copy(args[argnum])))
            if argnum not in expanded:
                expanded[argnum] = self.expand_macros(copy.copy(args[argnum]))
            rep[i:i+1] = expanded[argnum]

    # Get rid of removed comma if necessary
    if comma_patch:
        rep = [_i for _i in rep if _i]
        
    # Do a token concatenation pass, stitching any tokens separated by ## into a single token
    while len(rep) and rep[0].type == self.t_DPOUND:
        del rep[0]
    while len(rep) and rep[-1].type == self.t_DPOUND:
        del rep[-1]
    i = 1
    stitched = False
    while i < len(rep) - 1:
        if rep[i].type == self.t_DPOUND:
            j = i + 1
            while rep[j].type == self.t_DPOUND:
                j += 1
            rep[i-1] = copy.copy(rep[i-1])
            rep[i-1].type = None
            rep[i-1].value += rep[j].value
            while j >= i:
                del rep[i]
                j -= 1
            stitched = True
        else:
            i += 1
    if stitched:
        # Stitched tokens will have unknown type, so figure those out now
        i = 0
        lex = self.lexer.clone()
        while i < len(rep):
            if rep[i].type is None:
                lex.input(rep[i].value)
                toks = []
                while True:
                    tok = lex.token()
                    if not tok:
                        break
                    toks.append(tok)
                if len(toks) != 1:
                    # Split it once again
                    while len(toks) > 1:
                        rep.insert(i+1, copy.copy(rep[i]))
                        rep[i+1].value = toks[-1].value
                        rep[i+1].type = toks[-1].type
                        toks.pop()
                    rep[i].value = toks[0].value
                    rep[i].type = toks[0].type
                else:
                    rep[i].type = toks[0].type
            i += 1

    #print rep
    return rep
def macro_prescan(self, macro)

Examine the macro value (token sequence) and identify patch points This is used to speed up macro expansion later on—we'll know right away where to apply patches to the value to form the expansion

Source code
def macro_prescan(self,macro):
    """Examine the macro value (token sequence) and identify patch points
    This is used to speed up macro expansion later on---we'll know
    right away where to apply patches to the value to form the expansion"""
    macro.patch     = []             # Standard macro arguments 
    macro.str_patch = []             # String conversion expansion
    macro.var_comma_patch = []       # Variadic macro comma patch
    i = 0
    #print("BEFORE", macro.value)
    #print("BEFORE", [x.value for x in macro.value])
    while i < len(macro.value):
        if macro.value[i].type == self.t_ID and macro.value[i].value in macro.arglist:
            argnum = macro.arglist.index(macro.value[i].value)
            # Conversion of argument to a string
            j = i - 1
            while j >= 0 and macro.value[j].type in self.t_WS:
                j -= 1
            if j >= 0 and macro.value[j].value == '#':
                macro.value[i] = copy.copy(macro.value[i])
                macro.value[i].type = self.t_STRING
                while i > j:
                    del macro.value[j]
                    i -= 1
                macro.str_patch.append((argnum,i))
                continue
            # Concatenation
            elif (i > 0 and macro.value[i-1].value == '##'):
                macro.patch.append(('t',argnum,i))
                i += 1
                continue
            elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'):
                macro.patch.append(('t',argnum,i))
                i += 1
                continue
            # Standard expansion
            else:
                macro.patch.append(('e',argnum,i))
        elif macro.value[i].value == '##':
            if macro.variadic and (i > 0) and (macro.value[i-1].value == ',') and \
                    ((i+1) < len(macro.value)) and (macro.value[i+1].type == self.t_ID) and \
                    (macro.value[i+1].value == macro.vararg):
                macro.var_comma_patch.append(i-1)
        i += 1
    macro.patch.sort(key=lambda x: x[2],reverse=True)
def parse(self, input, source=None, ignore={})

Parse input text.

Source code
def parse(self,input,source=None,ignore={}):
    """Parse input text."""
    if isinstance(input, FILE_TYPES):
        if source is None:
            source = input.name
        input = input.read()
    self.ignore = ignore
    self.parser = self.parsegen(input,source,os.path.abspath(source) if source else None)
    if source is not None:
        dname = os.path.dirname(source)
        self.temp_path.insert(0,dname)
def parsegen(self, input, source=None, abssource=None)

Parse an input string

Source code
def parsegen(self,input,source=None,abssource=None):
    """Parse an input string"""
    rewritten_source = source
    if abssource:
        rewritten_source = abssource
        for rewrite in self.rewrite_paths:
            temp = re.sub(rewrite[0], rewrite[1], rewritten_source)
            if temp != abssource:
                rewritten_source = temp
                if os.sep != '/':
                    rewritten_source = rewritten_source.replace(os.sep, '/')
                break

    # Replace trigraph sequences
    t = trigraph(input)
    lines = self.group_lines(t, rewritten_source)

    if not source:
        source = ""
    if not rewritten_source:
        rewritten_source = ""
        
    my_include_times_idx = len(self.include_times)
    self.include_times.append(FileInclusionTime(self.macros['__FILE__'] if '__FILE__' in self.macros else None, source, abssource, self.include_depth))
    self.include_depth += 1
    my_include_time_begin = clock()
    if self.expand_filemacro:
        self.define("__FILE__ \"%s\"" % rewritten_source)

    self.source = abssource
    chunk = []
    enable = True
    iftrigger = False
    ifpassthru = False
    class ifstackentry(object):
        def __init__(self,enable,iftrigger,ifpassthru,startlinetoks):
            self.enable = enable
            self.iftrigger = iftrigger
            self.ifpassthru = ifpassthru
            self.rewritten = False
            self.startlinetoks = startlinetoks
    ifstack = []
    # True until any non-whitespace output or anything with effects happens.
    at_front_of_file = True
    # True if auto pragma once still a possibility for this #include
    auto_pragma_once_possible = self.auto_pragma_once_enabled
    # =(MACRO, 0) means #ifndef MACRO or #if !defined(MACRO) seen, =(MACRO,1) means #define MACRO seen
    include_guard = None
    self.on_potential_include_guard(None)

    for x in lines:
        all_whitespace = True
        skip_auto_pragma_once_possible_check = False
        # Handle comments
        for i,tok in enumerate(x):
            if tok.type in self.t_COMMENT:
                if not self.on_comment(tok):
                    if tok.type == self.t_COMMENT1:
                        tok.value = ' '
                    elif tok.type == self.t_COMMENT2:
                        tok.value = '\n'
                    tok.type = 'CPP_WS'
        # Skip over whitespace
        for i,tok in enumerate(x):
            if tok.type not in self.t_WS and tok.type not in self.t_COMMENT:
                all_whitespace = False
                break
        output_and_expand_line = True
        output_unexpanded_line = False
        if tok.value == '#':
            precedingtoks = [ tok ]
            output_and_expand_line = False
            try:
                # Preprocessor directive      
                i += 1
                while i < len(x) and x[i].type in self.t_WS:
                    precedingtoks.append(x[i])
                    i += 1                    
                dirtokens = self.tokenstrip(x[i:])
                if dirtokens:
                    name = dirtokens[0].value
                    args = self.tokenstrip(dirtokens[1:])
                
                    if self.debugout is not None:
                        print("%d:%d:%d %s:%d #%s %s" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno, dirtokens[0].value, "".join([tok.value for tok in args])), file = self.debugout)
                        #print(ifstack)

                    handling = self.on_directive_handle(dirtokens[0],args,ifpassthru,precedingtoks)
                    assert handling == True or handling == None
                else:
                    name = ""
                    args = []
                    raise OutputDirective(Action.IgnoreAndRemove)
                    
                if name == 'define':
                    at_front_of_file = False
                    if enable:
                        for tok in self.expand_macros(chunk):
                            yield tok
                        chunk = []
                        if include_guard and include_guard[1] == 0:
                            if include_guard[0] == args[0].value and len(args) == 1:
                                include_guard = (args[0].value, 1)
                                # If ifpassthru is only turned on due to this include guard, turn it off
                                if ifpassthru and not ifstack[-1].ifpassthru:
                                    ifpassthru = False
                        self.define(args)
                        if self.debugout is not None:
                            print("%d:%d:%d %s:%d      %s" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno, repr(self.macros[args[0].value])), file = self.debugout)
                        if handling is None:
                            for tok in x:
                                yield tok
                elif name == 'include':
                    if enable:
                        for tok in self.expand_macros(chunk):
                            yield tok
                        chunk = []
                        oldfile = self.macros['__FILE__'] if '__FILE__' in self.macros else None
                        if args and args[0].value != '<' and args[0].type != self.t_STRING:
                            args = self.tokenstrip(self.expand_macros(args))
                        #print('***', ''.join([x.value for x in args]), file = sys.stderr)
                        if self.passthru_includes is not None and self.passthru_includes.match(''.join([x.value for x in args])):
                            for tok in precedingtoks:
                                yield tok
                            for tok in dirtokens:
                                yield tok
                            for tok in self.include(args):
                                pass
                        else:
                            for tok in self.include(args):
                                yield tok
                        if oldfile is not None:
                            self.macros['__FILE__'] = oldfile
                        self.source = abssource
                elif name == 'undef':
                    at_front_of_file = False
                    if enable:
                        for tok in self.expand_macros(chunk):
                            yield tok
                        chunk = []
                        self.undef(args)
                        if handling is None:
                            for tok in x:
                                yield tok
                elif name == 'ifdef':
                    at_front_of_file = False
                    ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x))
                    if enable:
                        ifpassthru = False
                        if not args[0].value in self.macros:
                            res = self.on_unknown_macro_in_defined_expr(args[0])
                            if res is None:
                                ifpassthru = True
                                ifstack[-1].rewritten = True
                                raise OutputDirective(Action.IgnoreAndPassThrough)
                            elif res is True:
                                iftrigger = True
                            else:
                                enable = False
                                iftrigger = False
                        else:
                            iftrigger = True
                elif name == 'ifndef':
                    if not ifstack and at_front_of_file:
                        self.on_potential_include_guard(args[0].value)
                        include_guard = (args[0].value, 0)
                    at_front_of_file = False
                    ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x))
                    if enable:
                        ifpassthru = False
                        if args[0].value in self.macros:
                            enable = False
                            iftrigger = False
                        else:
                            res = self.on_unknown_macro_in_defined_expr(args[0])
                            if res is None:
                                ifpassthru = True
                                ifstack[-1].rewritten = True
                                raise OutputDirective(Action.IgnoreAndPassThrough)
                            elif res is True:
                                enable = False
                                iftrigger = False
                            else:
                                iftrigger = True
                elif name == 'if':
                    if not ifstack and at_front_of_file:
                        if args[0].value == '!' and args[1].value == 'defined':
                            n = 2
                            if args[n].value == '(': n += 1
                            self.on_potential_include_guard(args[n].value)
                            include_guard = (args[n].value, 0)
                    at_front_of_file = False
                    ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x))
                    if enable:
                        iftrigger = False
                        ifpassthru = False
                        result, rewritten = self.evalexpr(args)
                        if rewritten is not None:
                            x = x[:i+2] + rewritten + [x[-1]]
                            x[i+1] = copy.copy(x[i+1])
                            x[i+1].type = self.t_SPACE
                            x[i+1].value = ' '
                            ifpassthru = True
                            ifstack[-1].rewritten = True
                            raise OutputDirective(Action.IgnoreAndPassThrough)
                        if not result:
                            enable = False
                        else:
                            iftrigger = True
                elif name == 'elif':
                    at_front_of_file = False
                    if ifstack:
                        if ifstack[-1].enable:     # We only pay attention if outer "if" allows this
                            if enable and not ifpassthru:         # If already true, we flip enable False
                                enable = False
                            elif not iftrigger:   # If False, but not triggered yet, we'll check expression
                                result, rewritten = self.evalexpr(args)
                                if rewritten is not None:
                                    enable = True
                                    if not ifpassthru:
                                        # This is a passthru #elif after a False #if, so convert to an #if
                                        x[i].value = 'if'
                                    x = x[:i+2] + rewritten + [x[-1]]
                                    x[i+1] = copy.copy(x[i+1])
                                    x[i+1].type = self.t_SPACE
                                    x[i+1].value = ' '
                                    ifpassthru = True
                                    ifstack[-1].rewritten = True
                                    raise OutputDirective(Action.IgnoreAndPassThrough)
                                if ifpassthru:
                                    # If this elif can only ever be true, simulate that
                                    if result:
                                        newtok = copy.copy(x[i+3])
                                        newtok.type = self.t_INTEGER
                                        newtok.value = self.t_INTEGER_TYPE(result)
                                        x = x[:i+2] + [newtok] + [x[-1]]
                                        raise OutputDirective(Action.IgnoreAndPassThrough)
                                    # Otherwise elide
                                    enable = False
                                elif result:
                                    enable  = True
                                    iftrigger = True
                    else:
                        self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #elif")
                        
                elif name == 'else':
                    at_front_of_file = False
                    if ifstack:
                        if ifstack[-1].enable:
                            if ifpassthru:
                                enable = True
                                raise OutputDirective(Action.IgnoreAndPassThrough)
                            if enable:
                                enable = False
                            elif not iftrigger:
                                enable = True
                                iftrigger = True
                    else:
                        self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #else")

                elif name == 'endif':
                    at_front_of_file = False
                    if ifstack:
                        oldifstackentry = ifstack.pop()
                        enable = oldifstackentry.enable
                        iftrigger = oldifstackentry.iftrigger
                        ifpassthru = oldifstackentry.ifpassthru
                        if self.debugout is not None:
                            print("%d:%d:%d %s:%d      (%s:%d %s)" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno,
                                oldifstackentry.startlinetoks[0].source, oldifstackentry.startlinetoks[0].lineno, "".join([n.value for n in oldifstackentry.startlinetoks])), file = self.debugout)
                        skip_auto_pragma_once_possible_check = True
                        if oldifstackentry.rewritten:
                            raise OutputDirective(Action.IgnoreAndPassThrough)
                    else:
                        self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #endif")
                elif name == 'pragma' and args[0].value == 'once':
                    if enable:
                        self.include_once[self.source] = None
                elif enable:
                    # Unknown preprocessor directive
                    output_unexpanded_line = (self.on_directive_unknown(dirtokens[0], args, ifpassthru, precedingtoks) is None)

            except OutputDirective as e:
                if e.action == Action.IgnoreAndPassThrough:
                    output_unexpanded_line = True
                elif e.action == Action.IgnoreAndRemove:
                    pass
                else:
                    assert False

        # If there is ever any non-whitespace output outside an include guard, auto pragma once is not possible
        if not skip_auto_pragma_once_possible_check and auto_pragma_once_possible and not ifstack and not all_whitespace:
            auto_pragma_once_possible = False
            if self.debugout is not None:
                print("%d:%d:%d %s:%d Determined that #include \"%s\" is not entirely wrapped in an include guard macro, disabling auto-applying #pragma once" % (enable, iftrigger, ifpassthru, x[0].source, x[0].lineno, self.source), file = self.debugout)
            
        if output_and_expand_line or output_unexpanded_line:
            if not all_whitespace:
                at_front_of_file = False

            # Normal text
            if enable:
                if output_and_expand_line:
                    chunk.extend(x)
                elif output_unexpanded_line:
                    for tok in self.expand_macros(chunk):
                        yield tok
                    chunk = []
                    for tok in x:
                        yield tok
            else:
                # Need to extend with the same number of blank lines
                i = 0
                while i < len(x):
                    if x[i].type not in self.t_WS:
                        del x[i]
                    else:
                        i += 1
                chunk.extend(x)

    for tok in self.expand_macros(chunk):
        yield tok
    chunk = []
    for i in ifstack:
        self.on_error(i.startlinetoks[0].source, i.startlinetoks[0].lineno, "Unterminated " + "".join([n.value for n in i.startlinetoks]))
    if auto_pragma_once_possible and include_guard and include_guard[1] == 1:
        if self.debugout is not None:
            print("%d:%d:%d %s:%d Determined that #include \"%s\" is entirely wrapped in an include guard macro called %s, auto-applying #pragma once" % (enable, iftrigger, ifpassthru, self.source, 0, self.source, include_guard[0]), file = self.debugout)
        self.include_once[self.source] = include_guard[0]
    elif self.auto_pragma_once_enabled and self.source not in self.include_once:
        if self.debugout is not None:
            print("%d:%d:%d %s:%d Did not auto apply #pragma once to this file due to auto_pragma_once_possible=%d, include_guard=%s" % (enable, iftrigger, ifpassthru, self.source, 0, auto_pragma_once_possible, repr(include_guard)), file = self.debugout)
    my_include_time_end = clock()
    self.include_times[my_include_times_idx].elapsed = my_include_time_end - my_include_time_begin
    self.include_depth -= 1
def token(self)

Method to return individual tokens

Source code
def token(self):
    """Method to return individual tokens"""
    try:
        while True:
            tok = next(self.parser)
            if tok.type not in self.ignore:
                return tok
    except StopIteration:
        self.parser = None
        return None
def tokenize(self, text)

Utility function. Given a string of text, tokenize into a list of tokens

Source code
def tokenize(self,text):
    """Utility function. Given a string of text, tokenize into a list of tokens"""
    tokens = []
    self.lexer.input(text)
    while True:
        tok = self.lexer.token()
        if not tok: break
        tok.source = ''
        tokens.append(tok)
    return tokens
def tokenstrip(self, tokens)

Remove leading/trailing whitespace tokens from a token list

Source code
def tokenstrip(self,tokens):
    """Remove leading/trailing whitespace tokens from a token list"""
    i = 0
    while i < len(tokens) and tokens[i].type in self.t_WS:
        i += 1
    del tokens[:i]
    i = len(tokens)-1
    while i >= 0 and tokens[i].type in self.t_WS:
        i -= 1
    del tokens[i+1:]
    return tokens
def undef(self, tokens)

Undefine a macro

Source code
def undef(self,tokens):
    """Undefine a macro"""
    if isinstance(tokens,STRING_TYPES):
        tokens = self.tokenize(tokens)
    id = tokens[0].value
    try:
        del self.macros[id]
    except LookupError:
        pass
def write(self, oh=<_io.TextIOWrapper name='<stdout>' mode='w' encoding='utf-8'>)

Calls token() repeatedly, expanding tokens to their text and writing to the file like stream oh

Source code
def write(self, oh=sys.stdout):
    """Calls token() repeatedly, expanding tokens to their text and writing to the file like stream oh"""
    lastlineno = 0
    lastsource = None
    done = False
    blanklines = 0
    while not done:
        emitlinedirective = False
        toks = []
        all_ws = True
        # Accumulate a line
        while not done:
            tok = self.token()
            if not tok:
                done = True
                break
            toks.append(tok)
            if tok.value and tok.value[0] == '\n':
                break
            if tok.type not in self.t_WS:
                all_ws = False
        if not toks:
            break
        if all_ws:
            # Remove preceding whitespace so it becomes just a LF
            if len(toks) > 1:
                tok = toks[-1]
                toks = [ tok ]
            blanklines += toks[0].value.count('\n')
            continue
        # Filter out line continuations, collapsing before and after if needs be
        for n in xrange(len(toks)-1, -1, -1):
            if toks[n].type in self.t_LINECONT:
                if n > 0 and n < len(toks) - 1 and toks[n-1].type in self.t_WS and toks[n+1].type in self.t_WS:
                    toks[n-1].value = toks[n-1].value[0]
                    del toks[n:n+2]
                else:
                    del toks[n]
        # The line in toks is not all whitespace
        emitlinedirective = (blanklines > 6) and self.line_directive is not None
        if hasattr(toks[0], 'source'):
            if lastsource is None:
                if toks[0].source is not None:
                    emitlinedirective = True
                lastsource = toks[0].source
            elif lastsource != toks[0].source:
                emitlinedirective = True
                lastsource = toks[0].source
        # Replace consecutive whitespace in output with a single space except at any indent
        first_ws = None
        #print(toks)
        for n in xrange(len(toks)-1, -1, -1):
            tok = toks[n]
            if first_ws is None:
                if tok.type in self.t_SPACE or len(tok.value) == 0:
                    first_ws = n
            else:
                if tok.type not in self.t_SPACE and len(tok.value) > 0:
                    m = n + 1
                    while m != first_ws:
                        del toks[m]
                        first_ws -= 1
                    first_ws = None
                    if self.compress > 0:
                        # Collapse a token of many whitespace into single
                        if toks[m].value and toks[m].value[0] == ' ':
                            toks[m].value = ' '
        if not self.compress > 1 and not emitlinedirective:
            newlinesneeded = toks[0].lineno - lastlineno - 1
            if newlinesneeded > 6 and self.line_directive is not None:
                emitlinedirective = True
            else:
                while newlinesneeded > 0:
                    oh.write('\n')
                    newlinesneeded -= 1
        lastlineno = toks[0].lineno
        # Account for those newlines in a multiline comment
        if emitlinedirective and self.line_directive is not None:
            oh.write(self.line_directive + ' ' + str(lastlineno) + ('' if lastsource is None else (' "' + lastsource + '"' )) + '\n')
        for tok in toks:
            if tok.type == self.t_COMMENT1:
                lastlineno += tok.value.count('\n')
        blanklines = 0
        #print toks[0].lineno, 
        for tok in toks:
            #print tok.value,
            oh.write(tok.value)

Inherited members

class PreprocessorHooks

Override these in your subclass of Preprocessor to customise preprocessing

Source code
class PreprocessorHooks(object):
    """Override these in your subclass of Preprocessor to customise preprocessing"""
    def __init__(self):
        self.lastdirective = None

    def on_error(self,file,line,msg):
        """Called when the preprocessor has encountered an error, e.g. malformed input.
        
        The default simply prints to stderr and increments the return code.
        """
        print("%s:%d error: %s" % (file,line,msg), file = sys.stderr)
        self.return_code += 1
        
    def on_file_open(self,is_system_include,includepath):
        """Called to open a file for reading.
        
        This hook provides the ability to use ``chardet``, or any other mechanism,
        to inspect a file for its text encoding, and open it appropriately. Be
        aware that this function is used to probe for possible include file locations,
        so ``includepath`` may not exist. If it does not, raise the appropriate
        ``IOError`` exception.
        
        The default calls ``io.open(includepath, 'r', encoding = self.assume_encoding)``,
        examines if it starts with a BOM (if so, it removes it), and returns the file
        object opened. This raises the appropriate exception if the path was not found.
        """
        if sys.version_info.major < 3:
            assert self.assume_encoding is None
            ret = open(includepath, 'r')
        else:
            ret = open(includepath, 'r', encoding = self.assume_encoding)
        bom = ret.read(1)
        #print(repr(bom))
        if bom != '\ufeff':
            ret.seek(0)
        return ret

    def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath):
        """Called when a #include wasn't found.
        
        Raise OutputDirective to pass through or remove, else return
        a suitable path. Remember that Preprocessor.add_path() lets you add search paths.
        
        The default calls ``self.on_error()`` with a suitable error message about the
        include file not found if ``is_malformed`` is False, else a suitable error
        message about a malformed #include, and in both cases raises OutputDirective
        (pass through).
        """
        if is_malformed:
            self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Malformed #include statement: %s" % includepath)
        else:
            self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Include file '%s' not found" % includepath)
        raise OutputDirective(Action.IgnoreAndPassThrough)
        
    def on_unknown_macro_in_defined_expr(self,tok):
        """Called when an expression passed to an #if contained a defined operator
        performed on something unknown.
        
        Return True if to treat it as defined, False if to treat it as undefined,
        raise OutputDirective to pass through without execution, or return None to
        pass through the mostly expanded #if expression apart from the unknown defined.
        
        The default returns False, as per the C standard.
        """
        return False

    def on_unknown_macro_in_expr(self,ident):
        """Called when an expression passed to an #if contained an unknown identifier.
        
        Return what value the expression evaluator ought to use, or return None to
        pass through the mostly expanded #if expression.
        
        The default returns an integer 0, as per the C standard.
        """
        return 0
    
    def on_unknown_macro_function_in_expr(self,ident):
        """Called when an expression passed to an #if contained an unknown function.
        
        Return a callable which will be invoked by the expression evaluator to
        evaluate the input to the function, or return None to pass through the
        mostly expanded #if expression.
        
        The default returns a lambda which returns integer 0, as per the C standard.
        """
        return lambda x : 0
    
    def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks):
        """Called when there is one of
        
        define, include, undef, ifdef, ifndef, if, elif, else, endif
        
        Return True to execute and remove from the output, raise OutputDirective
        to pass through or remove without execution, or return None to execute
        AND pass through to the output (this only works for #define, #undef).
        
        The default returns True (execute and remove from the output).

        directive is the directive, toks is the tokens after the directive,
        ifpassthru is whether we are in passthru mode, precedingtoks is the
        tokens preceding the directive from the # token until the directive.
        """
        self.lastdirective = directive
        return True
        
    def on_directive_unknown(self,directive,toks,ifpassthru,precedingtoks):
        """Called when the preprocessor encounters a #directive it doesn't understand.
        This is actually quite an extensive list as it currently only understands:
        
        define, include, undef, ifdef, ifndef, if, elif, else, endif
        
        Return True to remove from the output, raise OutputDirective
        to pass through or remove, or return None to
        pass through into the output.
        
        The default handles #error and #warning by printing to stderr and returning True
        (remove from output). For everything else it returns None (pass through into output).

        directive is the directive, toks is the tokens after the directive,
        ifpassthru is whether we are in passthru mode, precedingtoks is the
        tokens preceding the directive from the # token until the directive.
        """
        if directive.value == 'error':
            print("%s:%d error: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
            self.return_code += 1
            return True
        elif directive.value == 'warning':
            print("%s:%d warning: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
            return True
        return None
        
    def on_potential_include_guard(self,macro):
        """Called when the preprocessor encounters an #ifndef macro or an #if !defined(macro)
        as the first non-whitespace thing in a file. Unlike the other hooks, macro is a string,
        not a token.
        """
        pass
    
    def on_comment(self,tok):
        """Called when the preprocessor encounters a comment token. You can modify the token
        in place. You must return True to let the comment pass through, else it will be removed.
        
        Returning False or None modifies the token to become whitespace, becoming a single space
        if the comment is a block comment, else a single new line if the comment is a line comment.
        """
        return None

Subclasses

Methods

def __init__(self)

Initialize self. See help(type(self)) for accurate signature.

Source code
def __init__(self):
    self.lastdirective = None
def on_comment(self, tok)

Called when the preprocessor encounters a comment token. You can modify the token in place. You must return True to let the comment pass through, else it will be removed.

Returning False or None modifies the token to become whitespace, becoming a single space if the comment is a block comment, else a single new line if the comment is a line comment.

Source code
def on_comment(self,tok):
    """Called when the preprocessor encounters a comment token. You can modify the token
    in place. You must return True to let the comment pass through, else it will be removed.
    
    Returning False or None modifies the token to become whitespace, becoming a single space
    if the comment is a block comment, else a single new line if the comment is a line comment.
    """
    return None
def on_directive_handle(self, directive, toks, ifpassthru, precedingtoks)

Called when there is one of

define, include, undef, ifdef, ifndef, if, elif, else, endif

Return True to execute and remove from the output, raise OutputDirective to pass through or remove without execution, or return None to execute AND pass through to the output (this only works for #define, #undef).

The default returns True (execute and remove from the output).

directive is the directive, toks is the tokens after the directive, ifpassthru is whether we are in passthru mode, precedingtoks is the tokens preceding the directive from the # token until the directive.

Source code
def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks):
    """Called when there is one of
    
    define, include, undef, ifdef, ifndef, if, elif, else, endif
    
    Return True to execute and remove from the output, raise OutputDirective
    to pass through or remove without execution, or return None to execute
    AND pass through to the output (this only works for #define, #undef).
    
    The default returns True (execute and remove from the output).

    directive is the directive, toks is the tokens after the directive,
    ifpassthru is whether we are in passthru mode, precedingtoks is the
    tokens preceding the directive from the # token until the directive.
    """
    self.lastdirective = directive
    return True
def on_directive_unknown(self, directive, toks, ifpassthru, precedingtoks)

Called when the preprocessor encounters a #directive it doesn't understand. This is actually quite an extensive list as it currently only understands:

define, include, undef, ifdef, ifndef, if, elif, else, endif

Return True to remove from the output, raise OutputDirective to pass through or remove, or return None to pass through into the output.

The default handles #error and #warning by printing to stderr and returning True (remove from output). For everything else it returns None (pass through into output).

directive is the directive, toks is the tokens after the directive, ifpassthru is whether we are in passthru mode, precedingtoks is the tokens preceding the directive from the # token until the directive.

Source code
def on_directive_unknown(self,directive,toks,ifpassthru,precedingtoks):
    """Called when the preprocessor encounters a #directive it doesn't understand.
    This is actually quite an extensive list as it currently only understands:
    
    define, include, undef, ifdef, ifndef, if, elif, else, endif
    
    Return True to remove from the output, raise OutputDirective
    to pass through or remove, or return None to
    pass through into the output.
    
    The default handles #error and #warning by printing to stderr and returning True
    (remove from output). For everything else it returns None (pass through into output).

    directive is the directive, toks is the tokens after the directive,
    ifpassthru is whether we are in passthru mode, precedingtoks is the
    tokens preceding the directive from the # token until the directive.
    """
    if directive.value == 'error':
        print("%s:%d error: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
        self.return_code += 1
        return True
    elif directive.value == 'warning':
        print("%s:%d warning: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr)
        return True
    return None
def on_error(self, file, line, msg)

Called when the preprocessor has encountered an error, e.g. malformed input.

The default simply prints to stderr and increments the return code.

Source code
def on_error(self,file,line,msg):
    """Called when the preprocessor has encountered an error, e.g. malformed input.
    
    The default simply prints to stderr and increments the return code.
    """
    print("%s:%d error: %s" % (file,line,msg), file = sys.stderr)
    self.return_code += 1
def on_file_open(self, is_system_include, includepath)

Called to open a file for reading.

This hook provides the ability to use chardet, or any other mechanism, to inspect a file for its text encoding, and open it appropriately. Be aware that this function is used to probe for possible include file locations, so includepath may not exist. If it does not, raise the appropriate IOError exception.

The default calls io.open(includepath, 'r', encoding = self.assume_encoding), examines if it starts with a BOM (if so, it removes it), and returns the file object opened. This raises the appropriate exception if the path was not found.

Source code
def on_file_open(self,is_system_include,includepath):
    """Called to open a file for reading.
    
    This hook provides the ability to use ``chardet``, or any other mechanism,
    to inspect a file for its text encoding, and open it appropriately. Be
    aware that this function is used to probe for possible include file locations,
    so ``includepath`` may not exist. If it does not, raise the appropriate
    ``IOError`` exception.
    
    The default calls ``io.open(includepath, 'r', encoding = self.assume_encoding)``,
    examines if it starts with a BOM (if so, it removes it), and returns the file
    object opened. This raises the appropriate exception if the path was not found.
    """
    if sys.version_info.major < 3:
        assert self.assume_encoding is None
        ret = open(includepath, 'r')
    else:
        ret = open(includepath, 'r', encoding = self.assume_encoding)
    bom = ret.read(1)
    #print(repr(bom))
    if bom != '\ufeff':
        ret.seek(0)
    return ret
def on_include_not_found(self, is_malformed, is_system_include, curdir, includepath)

Called when a #include wasn't found.

Raise OutputDirective to pass through or remove, else return a suitable path. Remember that Preprocessor.add_path() lets you add search paths.

The default calls self.on_error() with a suitable error message about the include file not found if is_malformed is False, else a suitable error message about a malformed #include, and in both cases raises OutputDirective (pass through).

Source code
def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath):
    """Called when a #include wasn't found.
    
    Raise OutputDirective to pass through or remove, else return
    a suitable path. Remember that Preprocessor.add_path() lets you add search paths.
    
    The default calls ``self.on_error()`` with a suitable error message about the
    include file not found if ``is_malformed`` is False, else a suitable error
    message about a malformed #include, and in both cases raises OutputDirective
    (pass through).
    """
    if is_malformed:
        self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Malformed #include statement: %s" % includepath)
    else:
        self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Include file '%s' not found" % includepath)
    raise OutputDirective(Action.IgnoreAndPassThrough)
def on_potential_include_guard(self, macro)

Called when the preprocessor encounters an #ifndef macro or an #if !defined(macro) as the first non-whitespace thing in a file. Unlike the other hooks, macro is a string, not a token.

Source code
def on_potential_include_guard(self,macro):
    """Called when the preprocessor encounters an #ifndef macro or an #if !defined(macro)
    as the first non-whitespace thing in a file. Unlike the other hooks, macro is a string,
    not a token.
    """
    pass
def on_unknown_macro_function_in_expr(self, ident)

Called when an expression passed to an #if contained an unknown function.

Return a callable which will be invoked by the expression evaluator to evaluate the input to the function, or return None to pass through the mostly expanded #if expression.

The default returns a lambda which returns integer 0, as per the C standard.

Source code
def on_unknown_macro_function_in_expr(self,ident):
    """Called when an expression passed to an #if contained an unknown function.
    
    Return a callable which will be invoked by the expression evaluator to
    evaluate the input to the function, or return None to pass through the
    mostly expanded #if expression.
    
    The default returns a lambda which returns integer 0, as per the C standard.
    """
    return lambda x : 0
def on_unknown_macro_in_defined_expr(self, tok)

Called when an expression passed to an #if contained a defined operator performed on something unknown.

Return True if to treat it as defined, False if to treat it as undefined, raise OutputDirective to pass through without execution, or return None to pass through the mostly expanded #if expression apart from the unknown defined.

The default returns False, as per the C standard.

Source code
def on_unknown_macro_in_defined_expr(self,tok):
    """Called when an expression passed to an #if contained a defined operator
    performed on something unknown.
    
    Return True if to treat it as defined, False if to treat it as undefined,
    raise OutputDirective to pass through without execution, or return None to
    pass through the mostly expanded #if expression apart from the unknown defined.
    
    The default returns False, as per the C standard.
    """
    return False
def on_unknown_macro_in_expr(self, ident)

Called when an expression passed to an #if contained an unknown identifier.

Return what value the expression evaluator ought to use, or return None to pass through the mostly expanded #if expression.

The default returns an integer 0, as per the C standard.

Source code
def on_unknown_macro_in_expr(self,ident):
    """Called when an expression passed to an #if contained an unknown identifier.
    
    Return what value the expression evaluator ought to use, or return None to
    pass through the mostly expanded #if expression.
    
    The default returns an integer 0, as per the C standard.
    """
    return 0
================================================ FILE: pcpp/__init__.py ================================================ from .evaluator import Evaluator from .parser import Action, OutputDirective from .pcmd import main, version, CmdPreprocessor from .preprocessor import Preprocessor __version__ = version ================================================ FILE: pcpp/evaluator.py ================================================ #!/usr/bin/python # Python C99 conforming preprocessor expression evaluator # (C) 2019-2026 Niall Douglas http://www.nedproductions.biz/ # Started: Apr 2019 import sys, os, re, codecs, copy if __name__ == '__main__' and __package__ is None: sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) ) from pcpp.parser import STRING_TYPES, yacc, default_lexer, in_production # The width of signed integer which this evaluator will use INTMAXBITS = 64 INTBASETYPE = int # Precompile the regular expression for correctly expanding unicode escape # sequences in Python 2 and 3. See https://stackoverflow.com/questions/4020539/process-escape-sequences-in-a-string-in-python # for more information. _expand_escape_sequences_pat = re.compile(r''' ( \\U........ # 8-digit hex escapes | \\u.... # 4-digit hex escapes | \\x.. # 2-digit hex escapes | \\[0-7]{1,3} # Octal escapes | \\N\{[^}]+\} # Unicode characters by name | \\[\\'"abfnrtv] # Single-character escapes )''', re.UNICODE | re.VERBOSE) class Value(INTBASETYPE): """A signed or unsigned integer within a preprocessor expression, bounded to within INT_MIN and INT_MAX, or 0 and UINT_MAX. Signed overflow is handled like a two's complement CPU, despite being UB, as that's what GCC and clang do. >>> Value(5) Value(5) >>> Value('5L') Value(5) >>> Value('5U') Value(5U) >>> Value('0') Value(0) >>> Value('0U') Value(0U) >>> Value('-1U') Value(18446744073709551615U) >>> Value(5) * Value(2) Value(10) >>> Value(5) + Value('2u') Value(7U) >>> Value(5) * 2 Value(10) >>> Value(5) / 2 # Must return integer Value(2) >>> Value(50) % 8 Value(2) >>> -Value(5) Value(-5) >>> +Value(-5) Value(-5) >>> ~Value(5) Value(-6) >>> Value(6) & 2 Value(2) >>> Value(4) | 2 Value(6) >>> Value(6) ^ 2 Value(4) >>> Value(2) << 2 Value(8) >>> Value(8) >> 2 Value(2) >>> Value(9223372036854775808) Value(-9223372036854775808) >>> Value(-9223372036854775809) Value(9223372036854775807) >>> Value(18446744073709551615) Value(-1) >>> Value(False) Value(0) >>> Value(True) Value(1) >>> Value(5) == Value(6) Value(0) >>> Value(5) == Value(5) Value(1) >>> not Value(2) Traceback (most recent call last): ... AssertionError >>> Value(4) and Value(2) Traceback (most recent call last): ... AssertionError >>> Value(5) and not Value(6) Traceback (most recent call last): ... AssertionError >>> Value('0x3f') Value(63) >>> Value('077') Value(63) >>> Value("'N'") Value(78) >>> Value("L'N'") Value(78) >>> Value("u8'N'") Value(78) >>> Value("u'N'") Value(78) >>> Value("U'N'") Value(78) >>> Value("u'猫'") Value(29483) >>> Value("U'猫'") Value(29483) >>> Value("U'🍌'") Value(127820) >>> Value("'\\n'") Value(10) >>> Value("'\\\\n'") Value(10) >>> Value("'\\\\'") Value(92) >>> Value("'\\'") Traceback (most recent call last): ... SyntaxError: Empty character escape sequence """ INT_MIN = -(1 << (INTMAXBITS - 1)) INT_MAX = (1 << (INTMAXBITS - 1)) - 1 INT_MASK = (1 << INTMAXBITS) - 1 UINT_MIN = 0 UINT_MAX = (1 << INTMAXBITS) - 1 @classmethod def __sclamp(cls, value): value = INTBASETYPE(value) return ((value - cls.INT_MIN) & cls.INT_MASK) + cls.INT_MIN @classmethod def __uclamp(cls, value): value = INTBASETYPE(value) return value & cls.UINT_MAX def __new__(cls, value, unsigned = False, exception = None): if isinstance(value, Value): unsigned = value.unsigned exception = value.exception elif isinstance(value, INTBASETYPE) or isinstance(value, int) or isinstance(value, float): value = cls.__uclamp(value) if unsigned else cls.__sclamp(value) elif isinstance(value, STRING_TYPES): if (value.startswith("L'") or value.startswith("u8'") or value.startswith("u'") or value.startswith("U'") or value[0] == "'") and value[-1] == "'": startidx = 3 if value.startswith("u8") else 2 if value[0] != "'" else 1 #print("1. ***", value, file = sys.stderr) value = value[startidx:-1].replace("\\\n", '') if len(value) == 0: raise SyntaxError('Empty character escape sequence') #print("2. ***", value, file = sys.stderr) value = _expand_escape_sequences_pat.sub(lambda x: codecs.decode(x.group(0), 'unicode-escape'), value) #print("3. ***", value, file = sys.stderr) x = INTBASETYPE(ord(value)) #print("4. ***", x, file = sys.stderr) elif value.startswith('0x') or value.startswith('0X'): # Strip any terminators while not ((value[-1] >= '0' and value[-1] <= '9') or (value[-1] >= 'a' and value[-1] <= 'f') or (value[-1] >= 'A' and value[-1] <= 'F')): if value[-1] == 'u' or value[-1] == 'U': unsigned = True value = value[:-1] x = INTBASETYPE(value, base = 16) elif value.startswith('0'): # Strip any terminators while not (value[-1] >= '0' and value[-1] <= '7'): if value[-1] == 'u' or value[-1] == 'U': unsigned = True value = value[:-1] x = INTBASETYPE(value, base = 8) else: # Strip any terminators while not (value[-1] >= '0' and value[-1] <= '9'): if value[-1] == 'u' or value[-1] == 'U': unsigned = True value = value[:-1] x = INTBASETYPE(value) value = cls.__uclamp(x) if unsigned else cls.__sclamp(x) #assert x == value else: print('Unknown value type: %s' % repr(type(value)), file = sys.stderr) assert False # Input is an unrecognised type inst = super(Value, cls).__new__(cls, value) inst.unsigned = unsigned inst.exception = exception return inst def value(self): if self.exception is not None: raise self.exception return INTBASETYPE(self) def __add__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) + self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__add__(other)) def __sub__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) - self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__sub__(other)) def __mul__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) * self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__mul__(other)) def __div__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) / self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__div__(other)) def __truediv__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) / self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__truediv__(other)) def __mod__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) % self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__mod__(other)) def __neg__(self): if self.exception is not None: return self return Value(super(Value, self).__neg__(), self.unsigned) def __invert__(self): if self.exception is not None: return self return Value(super(Value, self).__invert__(), self.unsigned) def __and__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) & self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__and__(other)) def __or__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) | self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__or__(other)) def __pos__(self): if self.exception is not None: return self return Value(super(Value, self).__pos__()) def __pow__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) ** self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__pow__(other)) def __lshift__(self, other): if self.exception is not None: return self # Ignore other signedness other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) << self.__uclamp(other), True) if (self.unsigned) else Value(super(Value, self).__lshift__(other)) def __rshift__(self, other): if self.exception is not None: return self # Ignore other signedness other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) >> self.__uclamp(other), True) if (self.unsigned) else Value(super(Value, self).__rshift__(other)) def __xor__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) ^ self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(super(Value, self).__xor__(other)) def __repr__(self): if self.exception is not None: return "Exception(%s)" % repr(self.exception) elif self.unsigned: return "Value(%dU)" % INTBASETYPE(self) else: return "Value(%d)" % INTBASETYPE(self) def __bool__(self): assert False # Do not use Python logical operations def __cmp__(self, other): assert False def __lt__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) < self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) < self.__sclamp(other), False) def __le__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) <= self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) <= self.__sclamp(other), False) def __eq__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) == self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) == self.__sclamp(other), False) def __ne__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) != self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) != self.__sclamp(other), False) def __ge__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) >= self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) >= self.__sclamp(other), False) def __gt__(self, other): if self.exception is not None: return self other = Value(other) if other.exception is not None: return other return Value(self.__uclamp(self) > self.__uclamp(other), True) if (self.unsigned or other.unsigned) else Value(self.__sclamp(self) > self.__sclamp(other), False) # PLY yacc specification # Valid C preprocessor expression items: # - Integer constants # - Character constants # - Addition, subtraction, multiplication, division, bitwise and-or-xor, shifts, # comparisons, logical and-or-not # - defined() # # The C preprocessor does not support: # - assignment # - increment and decrement # - array indexing, indirection # - casting # - sizeof, alignof # The subset of tokens from Preprocessor used in preprocessor expressions tokens = ( 'CPP_ID', 'PP_NUMBER', 'CPP_CHAR', 'CPP_STRING', 'CPP_PLUS', 'CPP_MINUS', 'CPP_STAR', 'CPP_FSLASH', 'CPP_PERCENT', 'CPP_BAR', 'CPP_AMPERSAND', 'CPP_TILDE', 'CPP_HAT', 'CPP_LESS', 'CPP_GREATER', 'CPP_EXCLAMATION', 'CPP_QUESTION', 'CPP_LPAREN', 'CPP_RPAREN', 'CPP_COMMA', 'CPP_COLON', 'CPP_LSHIFT', 'CPP_LESSEQUAL', 'CPP_RSHIFT', 'CPP_GREATEREQUAL', 'CPP_LOGICALOR', 'CPP_LOGICALAND', 'CPP_EQUALITY', 'CPP_INEQUALITY' ) # 'CPP_WS', 'CPP_EQUAL', 'CPP_BSLASH', 'CPP_SQUOTE', precedence = ( ('left', 'CPP_COMMA'), # 15 # 14 (assignments, unused) ('left', 'CPP_QUESTION', 'CPP_COLON'), # 13 ('left', 'CPP_LOGICALOR'), # 12 ('left', 'CPP_LOGICALAND'), # 11 ('left', 'CPP_BAR'), # 10 ('left', 'CPP_HAT'), # 9 ('left', 'CPP_AMPERSAND'), # 8 ('left', 'CPP_EQUALITY', 'CPP_INEQUALITY'), # 7 ('left', 'CPP_LESS', 'CPP_LESSEQUAL', 'CPP_GREATER', 'CPP_GREATEREQUAL'), # 6 ('left', 'CPP_LSHIFT', 'CPP_RSHIFT'), # 5 ('left', 'CPP_PLUS', 'CPP_MINUS'), # 4 ('left', 'CPP_STAR', 'CPP_FSLASH', 'CPP_PERCENT'), # 3 ('right', 'UPLUS', 'UMINUS', 'CPP_EXCLAMATION', 'CPP_TILDE'), # 2 # 1 (unused in the C preprocessor) ) def p_error(p): if p: raise SyntaxError("around token '%s' type %s" % (p.value, p.type)) else: raise SyntaxError("at EOF") def p_expression_number(p): 'expression : PP_NUMBER' try: p[0] = Value(p[1]) except: p[0] = p[1] def p_expression_character(p): 'expression : CPP_CHAR' p[0] = Value(p[1]) def p_expression_string(p): """ expression : CPP_STRING | CPP_LESS expression CPP_GREATER """ p[0] = p[1] def p_expression_group(t): 'expression : CPP_LPAREN expression CPP_RPAREN' t[0] = t[2] def p_expression_uplus(p): 'expression : CPP_PLUS expression %prec UPLUS' p[0] = +Value(p[2]) def p_expression_uminus(p): 'expression : CPP_MINUS expression %prec UMINUS' p[0] = -Value(p[2]) def p_expression_unop(p): """ expression : CPP_EXCLAMATION expression | CPP_TILDE expression """ try: if p[1] == '!': p[0] = Value(0) if (Value(p[2]).value()!=0) else Value(1) elif p[1] == '~': p[0] = ~Value(p[2]) except Exception as e: p[0] = Value(0, exception = e) def p_expression_binop(p): """ expression : expression CPP_STAR expression | expression CPP_FSLASH expression | expression CPP_PERCENT expression | expression CPP_PLUS expression | expression CPP_MINUS expression | expression CPP_LSHIFT expression | expression CPP_RSHIFT expression | expression CPP_LESS expression | expression CPP_LESSEQUAL expression | expression CPP_GREATER expression | expression CPP_GREATEREQUAL expression | expression CPP_EQUALITY expression | expression CPP_INEQUALITY expression | expression CPP_AMPERSAND expression | expression CPP_HAT expression | expression CPP_BAR expression | expression CPP_LOGICALAND expression | expression CPP_LOGICALOR expression | expression CPP_COMMA expression """ # print [repr(p[i]) for i in range(0,4)] try: if p[2] == '*': p[0] = Value(p[1]) * Value(p[3]) elif p[2] == '/': p[0] = Value(p[1]) / Value(p[3]) elif p[2] == '%': p[0] = Value(p[1]) % Value(p[3]) elif p[2] == '+': p[0] = Value(p[1]) + Value(p[3]) elif p[2] == '-': p[0] = Value(p[1]) - Value(p[3]) elif p[2] == '<<': p[0] = Value(p[1]) << Value(p[3]) elif p[2] == '>>': p[0] = Value(p[1]) >> Value(p[3]) elif p[2] == '<': p[0] = Value(p[1]) < Value(p[3]) elif p[2] == '<=': p[0] = Value(p[1]) <= Value(p[3]) elif p[2] == '>': p[0] = Value(p[1]) > Value(p[3]) elif p[2] == '>=': p[0] = Value(p[1]) >= Value(p[3]) elif p[2] == '==': p[0] = Value(p[1]) == Value(p[3]) elif p[2] == '!=': p[0] = Value(p[1]) != Value(p[3]) elif p[2] == '&': p[0] = Value(p[1]) & Value(p[3]) elif p[2] == '^': p[0] = Value(p[1]) ^ Value(p[3]) elif p[2] == '|': p[0] = Value(p[1]) | Value(p[3]) elif p[2] == '&&': p[0] = Value(1) if (Value(p[1]).value()!=0 and Value(p[3]).value()!=0) else Value(0) elif p[2] == '||': p[0] = Value(1) if (Value(p[1]).value()!=0 or Value(p[3]).value()!=0) else Value(0) elif p[2] == ',': p[0] = Value(p[3]) except Exception as e: p[0] = Value(0, exception = e) def p_expression_conditional(p): 'expression : expression CPP_QUESTION expression CPP_COLON expression' try: # Output type must cast up to unsigned if either input is unsigned p[0] = Value(p[3]) if (Value(p[1]).value()!=0) else Value(p[5]) try: p[0] = Value(p[0].value(), unsigned = Value(p[3]).unsigned or Value(p[5]).unsigned) except: pass except Exception as e: p[0] = Value(0, exception = e) def p_expression_function_call(p): "expression : CPP_ID CPP_LPAREN expression CPP_RPAREN" try: p.lexer.on_function_call(p) except Exception as e: p[0] = Value(0, exception = e) def p_expression_identifier(p): "expression : CPP_ID" try: p.lexer.on_identifier(p) except Exception as e: p[0] = Value(0, exception = e) class Evaluator(object): """Evaluator of #if C preprocessor expressions. >>> e = Evaluator() >>> e('5') Value(5) >>> e('5+6') Value(11) >>> e('5+6*2') Value(17) >>> e('5/2+6*2') Value(14) >>> e('5 < 6 <= 7') Value(1) >>> e('5 < 6 && 8 > 7') Value(1) >>> e('18446744073709551615 == -1') Value(1) >>> e('-9223372036854775809 == 9223372036854775807') Value(1) >>> e('-1 < 0U') Value(0U) >>> e('(( 0L && 0) || (!0L && !0 ))') Value(1) >>> e('(1)?2:3') Value(2) >>> e('(1 ? -1 : 0) <= 0') Value(1) >>> e('(1 ? -1 : 0U)') # Output type of ? must be common between both choices Value(18446744073709551615U) >>> e('(1 ? -1 : 0U) <= 0') Value(0U) >>> e('1 && 10 / 0') # doctest: +ELLIPSIS Exception(ZeroDivisionError(... >>> e('0 && 10 / 0') # && must shortcut Value(0) >>> e('1 ? 10 / 0 : 0') # doctest: +ELLIPSIS Exception(ZeroDivisionError(... >>> e('0 ? 10 / 0 : 0') # ? must shortcut Value(0) >>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1') Value(0) >>> e('1 << 2 != 4 || 8 >> 1 != 4') Value(0) >>> e('(2 || 3) != 1 || (2 && 3) != 1 || (0 || 4) != 1 || (0 && 5) != 0') Value(0) >>> e('-1 << 3U > 0') Value(0) >>> e("'N' == 78") Value(1) >>> e('0x3f == 63') Value(1) >>> e("'\\\\n'") Value(10) >>> e("'\\\\\\\\'") Value(92) >>> e("'\\\\n' == 0xA") Value(1) >>> e("'\\\\\\\\' == 0x5c") Value(1) >>> e("L'\\\\0' == 0") Value(1) >>> e('12 == 12') Value(1) >>> e('12L == 12') Value(1) >>> e('-1 >= 0U') Value(1U) >>> e('(1<<2) == 4') Value(1) >>> e('(-!+!9) == -1') Value(1) >>> e('(2 || 3) == 1') Value(1) >>> e('1L * 3 != 3') Value(0) >>> e('(!1L != 0) || (-1L != -1)') Value(0) >>> e('0177777 == 65535') Value(1) >>> e('0Xffff != 65535 || 0XFfFf == 65535') Value(1) >>> e('0L != 0 || 0l != 0') Value(0) >>> e('1U != 1 || 1u == 1') Value(1) >>> e('0 <= -1') Value(0) >>> e('1 << 2 != 4 || 8 >> 1 == 4') Value(1) >>> e('(3 ^ 5) == 6') Value(1) >>> e('(3 | 5) == 7') Value(1) >>> e('(3 & 5) == 1') Value(1) >>> e('(3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1') Value(0) >>> e('(0 ? 1 : 2) != 2') Value(0) >>> e('-1 << 3U > 0') Value(0) >>> e('0 && 10 / 0') Value(0) >>> e('not_defined && 10 / not_defined') # doctest: +ELLIPSIS Exception(SyntaxError('Unknown identifier not_defined'... >>> e('0 && 10 / 0 > 1') Value(0) >>> e('(0) ? 10 / 0 : 0') Value(0) >>> e('0 == 0 || 10 / 0 > 1') Value(1) >>> e('(15 >> 2 >> 1 != 1) || (3 << 2 << 1 != 24)') Value(0) >>> e('(1 | 2) == 3 && 4 != 5 || 0') Value(1) >>> e('1 > 0') Value(1) >>> e("'\123' != 83") Value(0) >>> e("'\x1b' != '\033'") Value(0) >>> e('0 + (1 - (2 + (3 - (4 + (5 - (6 + (7 - (8 + (9 - (10 + (11 - (12 + (13 - (14 + (15 - (16 + (17 - (18 + (19 - (20 + (21 - (22 + (23 - (24 + (25 - (26 + (27 - (28 + (29 - (30 + (31 - (32 + 0)))))))))) )))))))))))))))))))))) == 0') Value(1) >>> e('test_function(X)', functions={'test_function':lambda x: 55}) Value(55) >>> e('test_identifier', identifiers={'test_identifier':11}) Value(11) >>> e('defined(X)', functions={'defined':lambda x: 55}) Value(55) >>> e('defined(X)') # doctest: +ELLIPSIS Exception(SyntaxError('Unknown function defined'... >>> e('__has_include("variant")') # doctest: +ELLIPSIS Exception(SyntaxError('Unknown function __has_include'... >>> e('__has_include()') # doctest: +ELLIPSIS Exception(SyntaxError('Unknown function __has_include'... >>> e('5 // comment') Value(5) >>> e('5 /* comment */') Value(5) >>> e('5 /* comment // more */') Value(5) >>> e('5 // /* comment */') Value(5) """ # >>> e('defined X', functions={'defined':lambda x: 55}) # Value(55) def __init__(self, lexer = None): self.lexer = lexer if lexer is not None else default_lexer() self.parser = yacc.yacc(optimize=in_production,debug=not in_production,write_tables=not in_production) class __lexer(object): def __init__(self, functions, identifiers): self.__toks = [] self.__functions = functions self.__identifiers = identifiers def input(self, toks): self.__toks = [tok for tok in toks if tok.type != 'CPP_WS' and tok.type != 'CPP_LINECONT' and tok.type != 'CPP_COMMENT1' and tok.type != 'CPP_COMMENT2'] self.__idx = 0 def token(self): if self.__idx >= len(self.__toks): return None self.__idx = self.__idx + 1 return self.__toks[self.__idx - 1] def on_function_call(self, p): if p[1] not in self.__functions: raise SyntaxError('Unknown function %s' % p[1]) p[0] = Value(self.__functions[p[1]](p[3])) def on_identifier(self, p): if p[1] not in self.__identifiers: raise SyntaxError('Unknown identifier %s' % p[1]) p[0] = Value(self.__identifiers[p[1]]) def __call__(self, input, functions = {}, identifiers = {}): """Execute a fully macro expanded set of tokens representing an expression, returning the result of the evaluation. """ if not isinstance(input,list): self.lexer.input(input) input = [] while True: tok = self.lexer.token() if not tok: break input.append(tok) return self.parser.parse(input, lexer = self.__lexer(functions, identifiers)) if __name__ == "__main__": import doctest doctest.testmod() ================================================ FILE: pcpp/lextab.py ================================================ # lextab.py. This file automatically created by PLY (version 3.11). Don't edit! _tabversion = '3.10' _lextokens = set(('CPP_AMPERSAND', 'CPP_ANDEQUAL', 'CPP_BAR', 'CPP_BSLASH', 'CPP_CHAR', 'CPP_COLON', 'CPP_COMMA', 'CPP_COMMENT1', 'CPP_COMMENT2', 'CPP_DEREFERENCE', 'CPP_DIVIDEEQUAL', 'CPP_DOT', 'CPP_DPOUND', 'CPP_DQUOTE', 'CPP_EQUAL', 'CPP_EQUALITY', 'CPP_EXCLAMATION', 'CPP_FSLASH', 'CPP_GREATER', 'CPP_GREATEREQUAL', 'CPP_HAT', 'CPP_ID', 'CPP_INEQUALITY', 'CPP_LBRACKET', 'CPP_LCURLY', 'CPP_LESS', 'CPP_LESSEQUAL', 'CPP_LINECONT', 'CPP_LOGICALAND', 'CPP_LOGICALOR', 'CPP_LPAREN', 'CPP_LSHIFT', 'CPP_LSHIFTEQUAL', 'CPP_MINUS', 'CPP_MINUSEQUAL', 'CPP_MINUSMINUS', 'CPP_MULTIPLYEQUAL', 'CPP_OREQUAL', 'CPP_PERCENT', 'CPP_PERCENTEQUAL', 'CPP_PLUS', 'CPP_PLUSEQUAL', 'CPP_PLUSPLUS', 'CPP_POUND', 'CPP_QUESTION', 'CPP_RBRACKET', 'CPP_RCURLY', 'CPP_RPAREN', 'CPP_RSHIFT', 'CPP_RSHIFTEQUAL', 'CPP_SEMICOLON', 'CPP_SQUOTE', 'CPP_STAR', 'CPP_STRING', 'CPP_TILDE', 'CPP_WS', 'CPP_XOREQUAL', 'PP_NUMBER')) _lexreflags = 64 _lexliterals = '+-*/%|&~^<>=!?()[]{}.,;:\\\'"' _lexstateinfo = {'INITIAL': 'inclusive'} _lexstatere = {'INITIAL': [('(?P([ \\t]+|\\n))|(?P\\\\[ \\t]*\\n)|(?P\\.?\\d(?:\\.|[\\w_]|\'[\\w_]|[eEpP][-+])*)|(?P\\"([^\\\\\\n]|(\\\\(.|\\n)))*?\\")|(?P(u8|u|U|L)?\\\'([^\\\\\\n]|(\\\\(.|\\n)))*?\\\')|(?P(/\\*(.|\\n)*?\\*/))|(?P(//[^\\n]*))|(?P[A-Za-z_][\\w_]*)|(?P\\#\\#)|(?P\\|\\|)|(?P\\+\\+)|(?P\\|=)|(?P\\*=)|(?P\\+=)|(?P<<=)|(?P>>=)|(?P\\#)|(?P\\+)|(?P\\*)|(?P\\|)|(?P\\^)|(?P\\?)|(?P\\()|(?P\\))|(?P\\[)|(?P\\])|(?P\\.)|(?P\\\\)|(?P->)|(?P-=)|(?P--)|(?P<<)|(?P<=)|(?P>>)|(?P>=)|(?P&&)|(?P&=)|(?P==)|(?P!=)|(?P^=)|(?P/=)|(?P%=)|(?P-)|(?P/)|(?P%)|(?P&)|(?P~)|(?P<)|(?P>)|(?P=)|(?P!)|(?P{)|(?P})|(?P,)|(?P;)|(?P:)|(?P\')|(?P")', [None, ('t_CPP_WS', 'CPP_WS'), None, ('t_CPP_LINECONT', 'CPP_LINECONT'), ('t_PP_NUMBER', 'PP_NUMBER'), ('t_CPP_STRING', 'CPP_STRING'), None, None, None, ('t_CPP_CHAR', 'CPP_CHAR'), None, None, None, None, ('t_CPP_COMMENT1', 'CPP_COMMENT1'), None, None, ('t_CPP_COMMENT2', 'CPP_COMMENT2'), None, (None, 'CPP_ID'), (None, 'CPP_DPOUND'), (None, 'CPP_LOGICALOR'), (None, 'CPP_PLUSPLUS'), (None, 'CPP_OREQUAL'), (None, 'CPP_MULTIPLYEQUAL'), (None, 'CPP_PLUSEQUAL'), (None, 'CPP_LSHIFTEQUAL'), (None, 'CPP_RSHIFTEQUAL'), (None, 'CPP_POUND'), (None, 'CPP_PLUS'), (None, 'CPP_STAR'), (None, 'CPP_BAR'), (None, 'CPP_HAT'), (None, 'CPP_QUESTION'), (None, 'CPP_LPAREN'), (None, 'CPP_RPAREN'), (None, 'CPP_LBRACKET'), (None, 'CPP_RBRACKET'), (None, 'CPP_DOT'), (None, 'CPP_BSLASH'), (None, 'CPP_DEREFERENCE'), (None, 'CPP_MINUSEQUAL'), (None, 'CPP_MINUSMINUS'), (None, 'CPP_LSHIFT'), (None, 'CPP_LESSEQUAL'), (None, 'CPP_RSHIFT'), (None, 'CPP_GREATEREQUAL'), (None, 'CPP_LOGICALAND'), (None, 'CPP_ANDEQUAL'), (None, 'CPP_EQUALITY'), (None, 'CPP_INEQUALITY'), (None, 'CPP_XOREQUAL'), (None, 'CPP_DIVIDEEQUAL'), (None, 'CPP_PERCENTEQUAL'), (None, 'CPP_MINUS'), (None, 'CPP_FSLASH'), (None, 'CPP_PERCENT'), (None, 'CPP_AMPERSAND'), (None, 'CPP_TILDE'), (None, 'CPP_LESS'), (None, 'CPP_GREATER'), (None, 'CPP_EQUAL'), (None, 'CPP_EXCLAMATION'), (None, 'CPP_LCURLY'), (None, 'CPP_RCURLY'), (None, 'CPP_COMMA'), (None, 'CPP_SEMICOLON'), (None, 'CPP_COLON'), (None, 'CPP_SQUOTE'), (None, 'CPP_DQUOTE')])]} _lexstateignore = {'INITIAL': ''} _lexstateerrorf = {'INITIAL': 't_error'} _lexstateeoff = {} ================================================ FILE: pcpp/parser.py ================================================ #!/usr/bin/python # Python C99 conforming preprocessor parser config # (C) 2017-2026 Niall Douglas http://www.nedproductions.biz/ # and (C) 2007-2017 David Beazley http://www.dabeaz.com/ # Started: Feb 2017 # # This C preprocessor was originally written by David Beazley and the # original can be found at https://github.com/dabeaz/ply/blob/master/ply/cpp.py # This edition substantially improves on standards conforming output, # getting quite close to what clang or GCC outputs. import sys, re, os in_production = 1 # Set to 0 if editing pcpp implementation! STRING_TYPES = str # ----------------------------------------------------------------------------- # Default preprocessor lexer definitions. These tokens are enough to get # a basic preprocessor working. Other modules may import these if they want # ----------------------------------------------------------------------------- tokens = ( 'CPP_ID', 'PP_NUMBER', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_LINECONT', 'CPP_COMMENT1', 'CPP_COMMENT2', 'CPP_POUND','CPP_DPOUND', 'CPP_PLUS', 'CPP_MINUS', 'CPP_STAR', 'CPP_FSLASH', 'CPP_PERCENT', 'CPP_BAR', 'CPP_AMPERSAND', 'CPP_TILDE', 'CPP_HAT', 'CPP_LESS', 'CPP_GREATER', 'CPP_EQUAL', 'CPP_EXCLAMATION', 'CPP_QUESTION', 'CPP_LPAREN', 'CPP_RPAREN', 'CPP_LBRACKET', 'CPP_RBRACKET', 'CPP_LCURLY', 'CPP_RCURLY', 'CPP_DOT', 'CPP_COMMA', 'CPP_SEMICOLON', 'CPP_COLON', 'CPP_BSLASH', 'CPP_SQUOTE', 'CPP_DQUOTE', 'CPP_DEREFERENCE', 'CPP_MINUSEQUAL', 'CPP_MINUSMINUS', 'CPP_LSHIFT', 'CPP_LESSEQUAL', 'CPP_RSHIFT', 'CPP_GREATEREQUAL', 'CPP_LOGICALOR', 'CPP_OREQUAL', 'CPP_LOGICALAND', 'CPP_ANDEQUAL', 'CPP_EQUALITY', 'CPP_INEQUALITY', 'CPP_XOREQUAL', 'CPP_MULTIPLYEQUAL', 'CPP_DIVIDEEQUAL', 'CPP_PLUSEQUAL', 'CPP_PLUSPLUS', 'CPP_PERCENTEQUAL', 'CPP_LSHIFTEQUAL', 'CPP_RSHIFTEQUAL' ) literals = "+-*/%|&~^<>=!?()[]{}.,;:\\\'\"" # Whitespace, but don't match past the end of a line def t_CPP_WS(t): r'([ \t]+|\n)' t.lexer.lineno += t.value.count("\n") return t # Line continuation, accept whitespace between the backslash and new line def t_CPP_LINECONT(t): r'\\[ \t]*\n' t.value = t.value[1:-1] t.lexer.lineno += 1 return t _string_literal_linecont_pat = re.compile(r'\\[ \t]*\n') t_CPP_POUND = r'\#' t_CPP_DPOUND = r'\#\#' t_CPP_PLUS = r'\+' t_CPP_MINUS = r'-' t_CPP_STAR = r'\*' t_CPP_FSLASH = r'/' t_CPP_PERCENT = r'%' t_CPP_BAR = r'\|' t_CPP_AMPERSAND = r'&' t_CPP_TILDE = r'~' t_CPP_HAT = r'\^' t_CPP_LESS = r'<' t_CPP_GREATER = r'>' t_CPP_EQUAL = r'=' t_CPP_EXCLAMATION = r'!' t_CPP_QUESTION = r'\?' t_CPP_LPAREN = r'\(' t_CPP_RPAREN = r'\)' t_CPP_LBRACKET = r'\[' t_CPP_RBRACKET = r'\]' t_CPP_LCURLY = r'{' t_CPP_RCURLY = r'}' t_CPP_DOT = r'\.' t_CPP_COMMA = r',' t_CPP_SEMICOLON = r';' t_CPP_COLON = r':' t_CPP_BSLASH = r'\\' t_CPP_SQUOTE = r"'" t_CPP_DQUOTE = r'"' t_CPP_DEREFERENCE = r'->' t_CPP_MINUSEQUAL = r'-=' t_CPP_MINUSMINUS = r'--' t_CPP_LSHIFT = r'<<' t_CPP_LESSEQUAL = r'<=' t_CPP_RSHIFT = r'>>' t_CPP_GREATEREQUAL = r'>=' t_CPP_LOGICALOR = r'\|\|' t_CPP_OREQUAL = r'\|=' t_CPP_LOGICALAND = r'&&' t_CPP_ANDEQUAL = r'&=' t_CPP_EQUALITY = r'==' t_CPP_INEQUALITY = r'!=' t_CPP_XOREQUAL = r'^=' t_CPP_MULTIPLYEQUAL = r'\*=' t_CPP_DIVIDEEQUAL = r'/=' t_CPP_PLUSEQUAL = r'\+=' t_CPP_PLUSPLUS = r'\+\+' t_CPP_PERCENTEQUAL = r'%=' t_CPP_LSHIFTEQUAL = r'<<=' t_CPP_RSHIFTEQUAL = r'>>=' # Identifier t_CPP_ID = r'[A-Za-z_][\w_]*' # Preprocessor number def PP_NUMBER(t): r"\.?\d(?:\.|[\w_]|'[\w_]|[eEpP][-+])*" return t t_PP_NUMBER = PP_NUMBER # String literal def t_CPP_STRING(t): r'\"([^\\\n]|(\\(.|\n)))*?\"' t.value, subs_made = _string_literal_linecont_pat.subn('', t.value) t.lexer.lineno += subs_made + t.value.count("\n") return t # Character constant 'c' or L'c' or unicode editions thereof def t_CPP_CHAR(t): r'(u8|u|U|L)?\'([^\\\n]|(\\(.|\n)))*?\'' t.lexer.lineno += t.value.count("\n") return t # Comment def t_CPP_COMMENT1(t): r'(/\*(.|\n)*?\*/)' ncr = t.value.count("\n") t.lexer.lineno += ncr return t # Line comment def t_CPP_COMMENT2(t): r'(//[^\n]*)' return t def t_error(t): t.type = t.value[0] t.value = t.value[0] t.lexer.skip(1) return t # Python 2/3 compatible way of importing a subpackage oldsyspath = sys.path sys.path = [ os.path.join( os.path.dirname( os.path.abspath(__file__) ), "ply" ) ] + sys.path from ply import lex, yacc from ply.lex import LexToken sys.path = oldsyspath del oldsyspath # ----------------------------------------------------------------------------- # trigraph() # # Given an input string, this function replaces all trigraph sequences. # The following mapping is used: # # ??= # # ??/ \ # ??' ^ # ??( [ # ??) ] # ??! | # ??< { # ??> } # ??- ~ # ----------------------------------------------------------------------------- _trigraph_pat = re.compile(r'''\?\?[=/\'\(\)\!<>\-]''') _trigraph_rep = { '=':'#', '/':'\\', "'":'^', '(':'[', ')':']', '!':'|', '<':'{', '>':'}', '-':'~' } def trigraph(input): return _trigraph_pat.sub(lambda g: _trigraph_rep[g.group()[-1]],input) def default_lexer(): return lex.lex(optimize=in_production) # ------------------------------------------------------------------ # Macro object # # This object holds information about preprocessor macros # # .name - Macro name (string) # .value - Macro value (a list of tokens) # .arglist - List of argument names # .variadic - Boolean indicating whether or not variadic macro # .vararg - Name of the variadic parameter # # When a macro is created, the macro replacement token sequence is # pre-scanned and used to create patch lists that are later used # during macro expansion # ------------------------------------------------------------------ class Macro(object): def __init__(self,name,value,arglist=None,variadic=False): self.name = name self.value = value self.arglist = arglist self.variadic = variadic if variadic: self.vararg = arglist[-1] self.source = None self.lineno = None def __repr__(self): return "%s(%s)=%s" % (self.name, self.arglist, self.value) # ------------------------------------------------------------------ # Preprocessor event hooks # # Override these to customise preprocessing # ------------------------------------------------------------------ class Action(object): """What kind of abort processing to do in OutputDirective""" IgnoreAndPassThrough = 0 """Abort processing (don't execute), but pass the directive through to output""" IgnoreAndRemove = 1 """Abort processing (don't execute), and remove from output""" class OutputDirective(Exception): """Raise this exception to abort processing of a preprocessor directive and to instead output it as is into the output""" def __init__(self, action): self.action = action class PreprocessorHooks(object): """Override these in your subclass of Preprocessor to customise preprocessing""" def __init__(self): self.lastdirective = None def on_error(self,file,line,msg): """Called when the preprocessor has encountered an error, e.g. malformed input. The default simply prints to stderr and increments the return code. """ print("%s:%d error: %s" % (file,line,msg), file = sys.stderr) self.return_code += 1 def on_file_open(self,is_system_include,includepath): """Called to open a file for reading. This hook provides the ability to use ``chardet``, or any other mechanism, to inspect a file for its text encoding, and open it appropriately. Be aware that this function is used to probe for possible include file locations, so ``includepath`` may not exist. If it does not, raise the appropriate ``IOError`` exception. The default calls ``io.open(includepath, 'r', encoding = self.assume_encoding)``, examines if it starts with a BOM (if so, it removes it), and returns the file object opened. This raises the appropriate exception if the path was not found. """ ret = open(includepath, 'r', encoding = self.assume_encoding) bom = ret.read(1) #print(repr(bom)) if bom != '\ufeff': ret.seek(0) return ret def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath): """Called when a #include wasn't found. Raise OutputDirective to pass through or remove, else return a suitable path. Remember that Preprocessor.add_path() lets you add search paths. The default calls ``self.on_error()`` with a suitable error message about the include file not found if ``is_malformed`` is False, else a suitable error message about a malformed #include, and in both cases raises OutputDirective (pass through). """ if is_malformed: self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Malformed #include statement: %s" % includepath) else: self.on_error(self.lastdirective.source,self.lastdirective.lineno, "Include file '%s' not found" % includepath) raise OutputDirective(Action.IgnoreAndPassThrough) def on_unknown_macro_in_defined_expr(self,tok): """Called when an expression passed to an #if contained a defined operator performed on something unknown. Return True if to treat it as defined, False if to treat it as undefined, raise OutputDirective to pass through without execution, or return None to pass through the mostly expanded #if expression apart from the unknown defined. The default returns False, as per the C standard. """ return False def on_unknown_macro_in_expr(self,ident): """Called when an expression passed to an #if contained an unknown identifier. Return what value the expression evaluator ought to use, or return None to pass through the mostly expanded #if expression. The default returns an integer 0, as per the C standard. """ return 0 def on_unknown_macro_function_in_expr(self,ident): """Called when an expression passed to an #if contained an unknown function. Return a callable which will be invoked by the expression evaluator to evaluate the input to the function, or return None to pass through the mostly expanded #if expression. The default returns a lambda which returns integer 0, as per the C standard. """ return lambda x : 0 def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks): """Called when there is one of define, include, undef, ifdef, ifndef, if, elif, else, endif Return True to execute and remove from the output, raise OutputDirective to pass through or remove without execution, or return None to execute AND pass through to the output (this only works for #define, #undef). The default returns True (execute and remove from the output). directive is the directive, toks is the tokens after the directive, ifpassthru is whether we are in passthru mode, precedingtoks is the tokens preceding the directive from the # token until the directive. """ self.lastdirective = directive return True def on_directive_unknown(self,directive,toks,ifpassthru,precedingtoks): """Called when the preprocessor encounters a #directive it doesn't understand. This is actually quite an extensive list as it currently only understands: define, include, undef, ifdef, ifndef, if, elif, else, endif Return True to remove from the output, raise OutputDirective to pass through or remove, or return None to pass through into the output. The default handles #error and #warning by printing to stderr and returning True (remove from output). For everything else it returns None (pass through into output). directive is the directive, toks is the tokens after the directive, ifpassthru is whether we are in passthru mode, precedingtoks is the tokens preceding the directive from the # token until the directive. """ if directive.value == 'error': print("%s:%d error: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr) self.return_code += 1 return True elif directive.value == 'warning': print("%s:%d warning: %s" % (directive.source,directive.lineno,''.join(tok.value for tok in toks)), file = sys.stderr) return True return None def on_potential_include_guard(self,macro): """Called when the preprocessor encounters an #ifndef macro or an #if !defined(macro) as the first non-whitespace thing in a file. Unlike the other hooks, macro is a string, not a token. """ pass def on_comment(self,tok): """Called when the preprocessor encounters a comment token. You can modify the token in place. You must return True to let the comment pass through, else it will be removed. Returning False or None modifies the token to become whitespace, becoming a single space if the comment is a block comment, else a single new line if the comment is a line comment. """ return None ================================================ FILE: pcpp/parsetab.py ================================================ # parsetab.py # This file is automatically generated. Do not edit. # pylint: disable=W,C,R _tabversion = '3.10' _lr_method = 'LALR' _lr_signature = 'leftCPP_COMMAleftCPP_QUESTIONCPP_COLONleftCPP_LOGICALORleftCPP_LOGICALANDleftCPP_BARleftCPP_HATleftCPP_AMPERSANDleftCPP_EQUALITYCPP_INEQUALITYleftCPP_LESSCPP_LESSEQUALCPP_GREATERCPP_GREATEREQUALleftCPP_LSHIFTCPP_RSHIFTleftCPP_PLUSCPP_MINUSleftCPP_STARCPP_FSLASHCPP_PERCENTrightUPLUSUMINUSCPP_EXCLAMATIONCPP_TILDECPP_AMPERSAND CPP_BAR CPP_CHAR CPP_COLON CPP_COMMA CPP_EQUALITY CPP_EXCLAMATION CPP_FSLASH CPP_GREATER CPP_GREATEREQUAL CPP_HAT CPP_ID CPP_INEQUALITY CPP_LESS CPP_LESSEQUAL CPP_LOGICALAND CPP_LOGICALOR CPP_LPAREN CPP_LSHIFT CPP_MINUS CPP_PERCENT CPP_PLUS CPP_QUESTION CPP_RPAREN CPP_RSHIFT CPP_STAR CPP_STRING CPP_TILDE PP_NUMBERexpression : PP_NUMBERexpression : CPP_CHAR\n expression : CPP_STRING\n | CPP_LESS expression CPP_GREATER\n expression : CPP_LPAREN expression CPP_RPARENexpression : CPP_PLUS expression %prec UPLUSexpression : CPP_MINUS expression %prec UMINUS\n expression : CPP_EXCLAMATION expression\n | CPP_TILDE expression\n \n expression : expression CPP_STAR expression\n | expression CPP_FSLASH expression\n | expression CPP_PERCENT expression\n | expression CPP_PLUS expression\n | expression CPP_MINUS expression\n | expression CPP_LSHIFT expression\n | expression CPP_RSHIFT expression\n | expression CPP_LESS expression\n | expression CPP_LESSEQUAL expression\n | expression CPP_GREATER expression\n | expression CPP_GREATEREQUAL expression\n | expression CPP_EQUALITY expression\n | expression CPP_INEQUALITY expression\n | expression CPP_AMPERSAND expression\n | expression CPP_HAT expression\n | expression CPP_BAR expression\n | expression CPP_LOGICALAND expression\n | expression CPP_LOGICALOR expression\n | expression CPP_COMMA expression\n expression : expression CPP_QUESTION expression CPP_COLON expressionexpression : CPP_ID CPP_LPAREN expression CPP_RPARENexpression : CPP_ID' _lr_action_items = {'PP_NUMBER':([0,5,6,7,8,9,10,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,38,59,62,],[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,]),'CPP_CHAR':([0,5,6,7,8,9,10,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,38,59,62,],[3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,]),'CPP_STRING':([0,5,6,7,8,9,10,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,38,59,62,],[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,]),'CPP_LESS':([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,],[5,19,-1,-2,-3,5,5,5,5,5,5,-31,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,19,19,-6,-7,-8,-9,5,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,19,19,19,19,19,19,19,19,19,-4,-5,19,5,-30,19,]),'CPP_LPAREN':([0,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,38,59,62,],[6,6,6,6,6,6,6,38,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,]),'CPP_PLUS':([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,],[7,15,-1,-2,-3,7,7,7,7,7,7,-31,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,15,15,-6,-7,-8,-9,7,-10,-11,-12,-13,-14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,7,-5,15,7,-30,15,]),'CPP_MINUS':([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,],[8,16,-1,-2,-3,8,8,8,8,8,8,-31,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,16,16,-6,-7,-8,-9,8,-10,-11,-12,-13,-14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,8,-5,16,8,-30,16,]),'CPP_EXCLAMATION':([0,5,6,7,8,9,10,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,38,59,62,],[9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,]),'CPP_TILDE':([0,5,6,7,8,9,10,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,38,59,62,],[10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,]),'CPP_ID':([0,5,6,7,8,9,10,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,38,59,62,],[11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,]),'$end':([1,2,3,4,11,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,59,60,63,64,],[0,-1,-2,-3,-31,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,-26,-27,-28,-4,-5,-30,-29,]),'CPP_STAR':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[12,-1,-2,-3,-31,12,12,-6,-7,-8,-9,-10,-11,-12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,-4,-5,12,-30,12,]),'CPP_FSLASH':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[13,-1,-2,-3,-31,13,13,-6,-7,-8,-9,-10,-11,-12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,-4,-5,13,-30,13,]),'CPP_PERCENT':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[14,-1,-2,-3,-31,14,14,-6,-7,-8,-9,-10,-11,-12,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,-4,-5,14,-30,14,]),'CPP_LSHIFT':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[17,-1,-2,-3,-31,17,17,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,17,17,17,17,17,17,17,17,17,17,17,17,17,-4,-5,17,-30,17,]),'CPP_RSHIFT':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[18,-1,-2,-3,-31,18,18,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,18,18,18,18,18,18,18,18,18,18,18,18,18,-4,-5,18,-30,18,]),'CPP_LESSEQUAL':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[20,-1,-2,-3,-31,20,20,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,20,20,20,20,20,20,20,20,20,-4,-5,20,-30,20,]),'CPP_GREATER':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[21,-1,-2,-3,-31,59,21,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,21,21,21,21,21,21,21,21,21,-4,-5,21,-30,21,]),'CPP_GREATEREQUAL':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[22,-1,-2,-3,-31,22,22,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,22,22,22,22,22,22,22,22,22,-4,-5,22,-30,22,]),'CPP_EQUALITY':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[23,-1,-2,-3,-31,23,23,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,23,23,23,23,23,23,23,-4,-5,23,-30,23,]),'CPP_INEQUALITY':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[24,-1,-2,-3,-31,24,24,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,24,24,24,24,24,24,24,-4,-5,24,-30,24,]),'CPP_AMPERSAND':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[25,-1,-2,-3,-31,25,25,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,25,25,25,25,25,25,-4,-5,25,-30,25,]),'CPP_HAT':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[26,-1,-2,-3,-31,26,26,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,26,26,26,26,26,-4,-5,26,-30,26,]),'CPP_BAR':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[27,-1,-2,-3,-31,27,27,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,27,27,27,27,-4,-5,27,-30,27,]),'CPP_LOGICALAND':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[28,-1,-2,-3,-31,28,28,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,-26,28,28,28,-4,-5,28,-30,28,]),'CPP_LOGICALOR':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[29,-1,-2,-3,-31,29,29,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,-26,-27,29,29,-4,-5,29,-30,29,]),'CPP_COMMA':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[30,-1,-2,-3,-31,30,30,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,-26,-27,-28,30,-4,-5,30,-30,-29,]),'CPP_QUESTION':([1,2,3,4,11,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,63,64,],[31,-1,-2,-3,-31,31,31,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,-26,-27,31,31,-4,-5,31,-30,-29,]),'CPP_RPAREN':([2,3,4,11,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,59,60,61,63,64,],[-1,-2,-3,-31,60,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,-26,-27,-28,-4,-5,63,-30,-29,]),'CPP_COLON':([2,3,4,11,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,63,64,],[-1,-2,-3,-31,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,-26,-27,-28,62,-4,-5,-30,-29,]),} _lr_action = {} for _k, _v in _lr_action_items.items(): for _x,_y in zip(_v[0],_v[1]): if not _x in _lr_action: _lr_action[_x] = {} _lr_action[_x][_k] = _y del _lr_action_items _lr_goto_items = {'expression':([0,5,6,7,8,9,10,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,38,59,62,],[1,32,33,34,35,36,37,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,61,48,64,]),} _lr_goto = {} for _k, _v in _lr_goto_items.items(): for _x, _y in zip(_v[0], _v[1]): if not _x in _lr_goto: _lr_goto[_x] = {} _lr_goto[_x][_k] = _y del _lr_goto_items _lr_productions = [ ("S' -> expression","S'",1,None,None,None), ('expression -> PP_NUMBER','expression',1,'p_expression_number','evaluator.py',397), ('expression -> CPP_CHAR','expression',1,'p_expression_character','evaluator.py',405), ('expression -> CPP_STRING','expression',1,'p_expression_string','evaluator.py',410), ('expression -> CPP_LESS expression CPP_GREATER','expression',3,'p_expression_string','evaluator.py',411), ('expression -> CPP_LPAREN expression CPP_RPAREN','expression',3,'p_expression_group','evaluator.py',416), ('expression -> CPP_PLUS expression','expression',2,'p_expression_uplus','evaluator.py',420), ('expression -> CPP_MINUS expression','expression',2,'p_expression_uminus','evaluator.py',424), ('expression -> CPP_EXCLAMATION expression','expression',2,'p_expression_unop','evaluator.py',429), ('expression -> CPP_TILDE expression','expression',2,'p_expression_unop','evaluator.py',430), ('expression -> expression CPP_STAR expression','expression',3,'p_expression_binop','evaluator.py',442), ('expression -> expression CPP_FSLASH expression','expression',3,'p_expression_binop','evaluator.py',443), ('expression -> expression CPP_PERCENT expression','expression',3,'p_expression_binop','evaluator.py',444), ('expression -> expression CPP_PLUS expression','expression',3,'p_expression_binop','evaluator.py',445), ('expression -> expression CPP_MINUS expression','expression',3,'p_expression_binop','evaluator.py',446), ('expression -> expression CPP_LSHIFT expression','expression',3,'p_expression_binop','evaluator.py',447), ('expression -> expression CPP_RSHIFT expression','expression',3,'p_expression_binop','evaluator.py',448), ('expression -> expression CPP_LESS expression','expression',3,'p_expression_binop','evaluator.py',449), ('expression -> expression CPP_LESSEQUAL expression','expression',3,'p_expression_binop','evaluator.py',450), ('expression -> expression CPP_GREATER expression','expression',3,'p_expression_binop','evaluator.py',451), ('expression -> expression CPP_GREATEREQUAL expression','expression',3,'p_expression_binop','evaluator.py',452), ('expression -> expression CPP_EQUALITY expression','expression',3,'p_expression_binop','evaluator.py',453), ('expression -> expression CPP_INEQUALITY expression','expression',3,'p_expression_binop','evaluator.py',454), ('expression -> expression CPP_AMPERSAND expression','expression',3,'p_expression_binop','evaluator.py',455), ('expression -> expression CPP_HAT expression','expression',3,'p_expression_binop','evaluator.py',456), ('expression -> expression CPP_BAR expression','expression',3,'p_expression_binop','evaluator.py',457), ('expression -> expression CPP_LOGICALAND expression','expression',3,'p_expression_binop','evaluator.py',458), ('expression -> expression CPP_LOGICALOR expression','expression',3,'p_expression_binop','evaluator.py',459), ('expression -> expression CPP_COMMA expression','expression',3,'p_expression_binop','evaluator.py',460), ('expression -> expression CPP_QUESTION expression CPP_COLON expression','expression',5,'p_expression_conditional','evaluator.py',506), ('expression -> CPP_ID CPP_LPAREN expression CPP_RPAREN','expression',4,'p_expression_function_call','evaluator.py',518), ('expression -> CPP_ID','expression',1,'p_expression_identifier','evaluator.py',525), ] ================================================ FILE: pcpp/pcmd.py ================================================ #!/usr/bin/python # Python C99 conforming preprocessor command line # (C) 2017-2026 Niall Douglas http://www.nedproductions.biz/ # Started: March 2017 import sys, argparse, traceback, os, copy, io, re if __name__ == '__main__' and __package__ is None: sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) ) from pcpp.preprocessor import Preprocessor, OutputDirective, Action version='1.31' __all__ = [] class FileAction(argparse.Action): def __init__(self, option_strings, dest, **kwargs): super(FileAction, self).__init__(option_strings, dest, **kwargs) def __call__(self, parser, namespace, values, option_string=None): if getattr(namespace, self.dest)[0] == sys.stdin: items = [] else: items = copy.copy(getattr(namespace, self.dest)) items += [argparse.FileType('rt')(value) for value in values] setattr(namespace, self.dest, items) class CmdPreprocessor(Preprocessor): def __init__(self, argv): if len(argv) < 2: argv = [argv[0], '--help'] argp = argparse.ArgumentParser(prog='pcpp', description= '''A pure universal Python C (pre-)preprocessor implementation very useful for pre-preprocessing header only C++ libraries into single file includes and other such build or packaging stage malarky.''', epilog= '''Note that so pcpp can stand in for other preprocessor tooling, it ignores any arguments it does not understand.''') argp.add_argument('inputs', metavar = 'input', default = [sys.stdin], nargs = '*', action = FileAction, help = 'Files to preprocess (use \'-\' for stdin)') argp.add_argument('-o', dest = 'output', metavar = 'path', type = argparse.FileType('wt'), default=sys.stdout, nargs = '?', help = 'Output to a file instead of stdout') argp.add_argument('-D', dest = 'defines', metavar = 'macro[=val]', nargs = 1, action = 'append', help = 'Predefine name as a macro [with value]') argp.add_argument('-U', dest = 'undefines', metavar = 'macro', nargs = 1, action = 'append', help = 'Pre-undefine name as a macro') argp.add_argument('-N', dest = 'nevers', metavar = 'macro', nargs = 1, action = 'append', help = 'Never define name as a macro, even if defined during the preprocessing.') argp.add_argument('-I', dest = 'includes', metavar = 'path', nargs = 1, action = 'append', help = "Path to search for unfound #include's") #argp.add_argument('--passthru', dest = 'passthru', action = 'store_true', help = 'Pass through everything unexecuted except for #include and include guards (which need to be the first thing in an include file') argp.add_argument('--passthru-defines', dest = 'passthru_defines', action = 'store_true', help = 'Pass through but still execute #defines and #undefs if not always removed by preprocessor logic') argp.add_argument('--passthru-unfound-includes', dest = 'passthru_unfound_includes', action = 'store_true', help = 'Pass through #includes not found without execution') argp.add_argument('--passthru-unknown-exprs', dest = 'passthru_undefined_exprs', action = 'store_true', help = 'Unknown macros in expressions cause preprocessor logic to be passed through instead of executed by treating unknown macros as 0L') argp.add_argument('--passthru-comments', dest = 'passthru_comments', action = 'store_true', help = 'Pass through comments unmodified') argp.add_argument('--passthru-magic-macros', dest = 'passthru_magic_macros', action = 'store_true', help = 'Pass through double underscore magic macros unmodified') argp.add_argument('--passthru-has-include', dest = 'passthru_has_include', action = 'store_true', help = 'Pass through __has_include expressions unmodified') argp.add_argument('--passthru-includes', dest = 'passthru_includes', metavar = '', default = None, nargs = 1, help = "Regular expression for which #includes to not expand. #includes, if found, are always executed") argp.add_argument('--disable-auto-pragma-once', dest = 'auto_pragma_once_disabled', action = 'store_true', default = False, help = 'Disable the heuristics which auto apply #pragma once to #include files wholly wrapped in an obvious include guard macro') argp.add_argument('--enable-include-next', dest = 'include_next_enabled', action = 'store_true', default = False, help = 'Enable #include_next support, which you should try to avoid using') argp.add_argument('--line-directive', dest = 'line_directive', metavar = 'form', default = '#line', nargs = '?', help = "Form of line directive to use, defaults to #line, specify nothing to disable output of line directives") argp.add_argument('--debug', dest = 'debug', action = 'store_true', help = 'Generate a pcpp_debug.log file logging execution') argp.add_argument('--time', dest = 'time', action = 'store_true', help = 'Print the time it took to #include each file') argp.add_argument('--filetimes', dest = 'filetimes', metavar = 'path', type = argparse.FileType('wt'), default=None, nargs = '?', help = 'Write CSV file with time spent inside each included file, inclusive and exclusive') argp.add_argument('--compress', dest = 'compress', action = 'store_true', help = 'Make output as small as possible') argp.add_argument('--assume-input-encoding', dest = 'assume_input_encoding', metavar = '', default = None, nargs = 1, help = 'The text encoding to assume inputs are in') argp.add_argument('--output-encoding', dest = 'output_encoding', metavar = '', default = None, nargs = 1, help = 'The text encoding to use when writing files') argp.add_argument('--write-bom', dest = 'write_bom', action = 'store_true', help = 'Prefix any output with a Unicode BOM') argp.add_argument('--trigraphs', dest = 'enable_trigraphs', action = 'store_true', help = 'Enable processing trigraphs') argp.add_argument('--version', action='version', version='pcpp ' + version) args = argp.parse_known_args(argv[1:]) #print(args) for arg in args[1]: print("NOTE: Argument %s not known, ignoring!" % arg, file = sys.stderr) self.args = args[0] super(CmdPreprocessor, self).__init__() # Override Preprocessor instance variables self.define("__PCPP_VERSION__ " + version) self.define("__PCPP_ALWAYS_FALSE__ 0") self.define("__PCPP_ALWAYS_TRUE__ 1") if self.args.debug: self.debugout = open("pcpp_debug.log", "wt") self.auto_pragma_once_enabled = not self.args.auto_pragma_once_disabled self.include_next_enabled = self.args.include_next_enabled self.line_directive = self.args.line_directive if self.line_directive is not None and self.line_directive.lower() in ('nothing', 'none', ''): self.line_directive = None if self.args.passthru_includes is not None: self.passthru_includes = re.compile(self.args.passthru_includes[0]) self.compress = 2 if self.args.compress else 0 if self.args.passthru_magic_macros: self.undef('__DATE__') self.undef('__TIME__') self.expand_linemacro = False self.expand_filemacro = False self.expand_countermacro = False self.passthru_expr_has_include = self.args.passthru_has_include if self.args.assume_input_encoding is not None: self.args.assume_input_encoding = self.args.assume_input_encoding[0] self.assume_encoding = self.args.assume_input_encoding if len(self.args.inputs) == 1: # Reopen our input files with the appropriate encoding _ = self.on_file_open(False, self.args.inputs[0].name) self.args.inputs[0].close() self.args.inputs[0] = _ if self.args.output_encoding is None: self.args.output_encoding = self.args.assume_input_encoding if self.args.output_encoding is not None: self.args.output_encoding = self.args.output_encoding[0] # Reopen our output file with the appropriate encoding _ = io.open(self.args.output.name, 'w', encoding = self.args.output_encoding) self.args.output.close() self.args.output = _ if self.args.write_bom: self.args.output.write('\ufeff') self.enable_trigraphs = self.args.enable_trigraphs # My own instance variables self.bypass_ifpassthru = False self.potential_include_guard = None if self.args.defines: self.args.defines = [x[0] for x in self.args.defines] for d in self.args.defines: if '=' not in d: d += '=1' d = d.replace('=', ' ', 1) self.define(d) if self.args.undefines: self.args.undefines = [x[0] for x in self.args.undefines] for d in self.args.undefines: self.undef(d) if self.args.nevers: self.args.nevers = [x[0] for x in self.args.nevers] if self.args.includes: self.args.includes = [x[0] for x in self.args.includes] for d in self.args.includes: self.add_path(d) try: if len(self.args.inputs) == 1: self.parse(self.args.inputs[0]) else: input = '' for i in self.args.inputs: input += '#include "' + i.name + '"\n' self.parse(input) self.write(self.args.output) except: print(traceback.print_exc(10), file = sys.stderr) print("\nINTERNAL PREPROCESSOR ERROR AT AROUND %s:%d, FATALLY EXITING NOW\n" % (self.lastdirective.source, self.lastdirective.lineno), file = sys.stderr) sys.exit(-99) finally: for i in self.args.inputs: i.close() if self.args.output != sys.stdout: self.args.output.close() if self.args.time: print("\nTime report:") print("============") for n in range(0, len(self.include_times)): if n == 0: print("top level: %f seconds" % self.include_times[n].elapsed) elif self.include_times[n].depth == 1: print("\n %s: %f seconds (%f%%)" % (self.include_times[n].included_path, self.include_times[n].elapsed, 100 * self.include_times[n].elapsed / self.include_times[0].elapsed)) else: print("%s%s: %f seconds" % (' ' * self.include_times[n].depth, self.include_times[n].included_path, self.include_times[n].elapsed)) print("\nPragma once files (including heuristically applied):") print("====================================================") for i in self.include_once: print(" ", i) print() if self.args.filetimes: print('"Total seconds","Self seconds","File size","File path"', file = self.args.filetimes) filetimes = {} currentfiles = [] for n in range(0, len(self.include_times)): while self.include_times[n].depth < len(currentfiles): currentfiles.pop() if self.include_times[n].depth > len(currentfiles) - 1: currentfiles.append(self.include_times[n].included_abspath) #print() #for path in currentfiles: # print("currentfiles =", path) path = currentfiles[-1] if path in filetimes: filetimes[path][0] += self.include_times[n].elapsed filetimes[path][1] += self.include_times[n].elapsed else: filetimes[path] = [self.include_times[n].elapsed, self.include_times[n].elapsed] if self.include_times[n].elapsed > 0 and len(currentfiles) > 1: #print("Removing child %f from parent %s = %f" % (self.include_times[n].elapsed, currentfiles[-2], filetimes[currentfiles[-2]])) filetimes[currentfiles[-2]][1] -= self.include_times[n].elapsed filetimes = [(v[0],v[1],k) for k,v in filetimes.items()] filetimes.sort(reverse=True) for t,s,p in filetimes: print(('%f,%f,%d,"%s"' % (t, s, os.stat(p).st_size, p)), file = self.args.filetimes) def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath): if self.args.passthru_unfound_includes: raise OutputDirective(Action.IgnoreAndPassThrough) return super(CmdPreprocessor, self).on_include_not_found(is_malformed,is_system_include,curdir,includepath) def on_unknown_macro_in_defined_expr(self,tok): if self.args.undefines: if tok.value in self.args.undefines: return False if self.args.passthru_undefined_exprs: return None # Pass through as expanded as possible return super(CmdPreprocessor, self).on_unknown_macro_in_defined_expr(tok) def on_unknown_macro_in_expr(self,ident): if self.args.undefines: if ident in self.args.undefines: return super(CmdPreprocessor, self).on_unknown_macro_in_expr(ident) if self.args.passthru_undefined_exprs: return None # Pass through as expanded as possible return super(CmdPreprocessor, self).on_unknown_macro_in_expr(ident) def on_unknown_macro_function_in_expr(self,ident): if self.args.undefines: if ident in self.args.undefines: return super(CmdPreprocessor, self).on_unknown_macro_function_in_expr(ident) if self.args.passthru_undefined_exprs: return None # Pass through as expanded as possible return super(CmdPreprocessor, self).on_unknown_macro_function_in_expr(ident) def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks): if ifpassthru: if directive.value == 'if' or directive.value == 'elif' or directive == 'else' or directive.value == 'endif': self.bypass_ifpassthru = len([tok for tok in toks if tok.value == '__PCPP_ALWAYS_FALSE__' or tok.value == '__PCPP_ALWAYS_TRUE__']) > 0 if not self.bypass_ifpassthru and (directive.value == 'define' or directive.value == 'undef'): if toks[0].value != self.potential_include_guard: raise OutputDirective(Action.IgnoreAndPassThrough) # Don't execute anything with effects when inside an #if expr with undefined macro if (directive.value == 'define' or directive.value == 'undef') and self.args.nevers: if toks[0].value in self.args.nevers: raise OutputDirective(Action.IgnoreAndPassThrough) if self.args.passthru_defines: super(CmdPreprocessor, self).on_directive_handle(directive,toks,ifpassthru,precedingtoks) return None # Pass through where possible return super(CmdPreprocessor, self).on_directive_handle(directive,toks,ifpassthru,precedingtoks) def on_directive_unknown(self,directive,toks,ifpassthru,precedingtoks): if ifpassthru: return None # Pass through return super(CmdPreprocessor, self).on_directive_unknown(directive,toks,ifpassthru,precedingtoks) def on_potential_include_guard(self,macro): self.potential_include_guard = macro return super(CmdPreprocessor, self).on_potential_include_guard(macro) def on_comment(self,tok): if self.args.passthru_comments: return True # Pass through return super(CmdPreprocessor, self).on_comment(tok) def main(argv=None): if argv is None: argv = sys.argv p = CmdPreprocessor(argv) return p.return_code if __name__ == "__main__": sys.exit(main(sys.argv)) ================================================ FILE: pcpp/preprocessor.py ================================================ #!/usr/bin/python # Python C99 conforming preprocessor useful for generating single include files # (C) 2017-2026 Niall Douglas http://www.nedproductions.biz/ # and (C) 2007-2017 David Beazley http://www.dabeaz.com/ # Started: Feb 2017 # # This C preprocessor was originally written by David Beazley and the # original can be found at https://github.com/dabeaz/ply/blob/master/ply/cpp.py # This edition substantially improves on standards conforming output, # getting quite close to what clang or GCC outputs. import sys, os, re, codecs, time, copy, traceback if __name__ == '__main__' and __package__ is None: sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) ) from pcpp.parser import STRING_TYPES, default_lexer, trigraph, Macro, Action, OutputDirective, PreprocessorHooks from pcpp.evaluator import Evaluator import io FILE_TYPES = io.IOBase clock = time.process_time __all__ = ['Preprocessor', 'PreprocessorHooks', 'OutputDirective', 'Action', 'Evaluator'] # ------------------------------------------------------------------ # File inclusion timings # # Useful for figuring out how long a sequence of preprocessor inclusions actually is # ------------------------------------------------------------------ class FileInclusionTime(object): """The seconds taken to #include another file""" def __init__(self,including_path,included_path,included_abspath,depth): self.including_path = including_path self.included_path = included_path self.included_abspath = included_abspath self.depth = depth self.elapsed = 0.0 # ------------------------------------------------------------------ # Preprocessor object # # Object representing a preprocessor. Contains macro definitions, # include directories, and other information # ------------------------------------------------------------------ class Preprocessor(PreprocessorHooks): def __init__(self,lexer=None): super(Preprocessor, self).__init__() if lexer is None: lexer = default_lexer() self.lexer = lexer self.evaluator = Evaluator(self.lexer) self.macros = { } self.path = [] # list of -I formal search paths for includes self.temp_path = [] # list of temporary search paths for includes self.rewrite_paths = [(re.escape(os.path.abspath('') + os.sep) + '(.*)', '\\1')] self.passthru_includes = None self.passthru_expr_has_include = False self.include_once = {} self.include_depth = 0 self.include_times = [] # list of FileInclusionTime self.return_code = 0 self.debugout = None self.auto_pragma_once_enabled = True self.include_next_enabled = False self.line_directive = '#line' self.compress = False self.assume_encoding = None self.enable_trigraphs = False # Probe the lexer for selected tokens self.__lexprobe() tm = time.localtime() self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm)) self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm)) self.define("__PCPP__ 1") self.expand_linemacro = True self.expand_filemacro = True self.expand_countermacro = True self.linemacro = 0 self.linemacrodepth = 0 self.countermacro = 0 self.current_include_next_unique_ids = [] self.parser = None @staticmethod def __file_unique_id(fh): s = os.stat(fh.fileno()) return s.st_ino ^ s.st_size # ----------------------------------------------------------------------------- # tokenize() # # Utility function. Given a string of text, tokenize into a list of tokens # ----------------------------------------------------------------------------- def tokenize(self,text): """Utility function. Given a string of text, tokenize into a list of tokens""" tokens = [] self.lexer.input(text) while True: tok = self.lexer.token() if not tok: break tok.source = '' tokens.append(tok) return tokens # ---------------------------------------------------------------------- # __lexprobe() # # This method probes the preprocessor lexer object to discover # the token types of symbols that are important to the preprocessor. # If this works right, the preprocessor will simply "work" # with any suitable lexer regardless of how tokens have been named. # ---------------------------------------------------------------------- def __lexprobe(self): # Determine the token type for identifiers self.lexer.input("identifier") tok = self.lexer.token() if not tok or tok.value != "identifier": print("Couldn't determine identifier type") else: self.t_ID = tok.type # Determine the token type for integers self.lexer.input("12345") tok = self.lexer.token() if not tok or int(tok.value) != 12345: print("Couldn't determine integer type") else: self.t_INTEGER = tok.type self.t_INTEGER_TYPE = type(tok.value) # Determine the token type for character self.lexer.input("'a'") tok = self.lexer.token() if not tok or tok.value != "'a'": print("Couldn't determine character type") else: self.t_CHAR = tok.type # Determine the token type for strings enclosed in double quotes self.lexer.input("\"filename\"") tok = self.lexer.token() if not tok or tok.value != "\"filename\"": print("Couldn't determine string type") else: self.t_STRING = tok.type # Determine the token type for whitespace--if any self.lexer.input(" ") tok = self.lexer.token() if not tok or tok.value != " ": self.t_SPACE = None else: self.t_SPACE = tok.type # Determine the token type for newlines self.lexer.input("\n") tok = self.lexer.token() if not tok or tok.value != "\n": self.t_NEWLINE = None print("Couldn't determine token for newlines") else: self.t_NEWLINE = tok.type # Determine the token type for line continuations self.lexer.input("\\ \n") tok = self.lexer.token() if not tok or tok.value != " ": self.t_LINECONT = None print("Couldn't determine token for line continuations") else: self.t_LINECONT = tok.type self.t_WS = (self.t_SPACE, self.t_NEWLINE, self.t_LINECONT) self.lexer.input("##") tok = self.lexer.token() if not tok or tok.value != "##": print("Couldn't determine token for token pasting operator") else: self.t_DPOUND = tok.type self.lexer.input("?") tok = self.lexer.token() if not tok or tok.value != "?": print("Couldn't determine token for ternary operator") else: self.t_TERNARY = tok.type self.lexer.input(":") tok = self.lexer.token() if not tok or tok.value != ":": print("Couldn't determine token for ternary operator") else: self.t_COLON = tok.type self.lexer.input("/* comment */") tok = self.lexer.token() if not tok or tok.value != "/* comment */": print("Couldn't determine comment type") else: self.t_COMMENT1 = tok.type self.lexer.input("// comment") tok = self.lexer.token() if not tok or tok.value != "// comment": print("Couldn't determine comment type") else: self.t_COMMENT2 = tok.type self.t_COMMENT = (self.t_COMMENT1, self.t_COMMENT2) # Check for other characters used by the preprocessor chars = [ '<','>','#','##','\\','(',')',',','.'] for c in chars: self.lexer.input(c) tok = self.lexer.token() if not tok or tok.value != c: print("Unable to lex '%s' required for preprocessor" % c) # ---------------------------------------------------------------------- # add_path() # # Adds a search path to the preprocessor. # ---------------------------------------------------------------------- def add_path(self,path): """Adds a search path to the preprocessor. """ self.path.append(path) # If the search path being added is relative, or has a common ancestor to the # current working directory, add a rewrite to relativise includes from this # search path relpath = None try: relpath = os.path.relpath(path) except: pass if relpath is not None: self.rewrite_paths += [(re.escape(os.path.abspath(path) + os.sep) + '(.*)', os.path.join(relpath, '\\1'))] # ---------------------------------------------------------------------- # group_lines() # # Given an input string, this function splits it into lines. Trailing whitespace # is removed. This function forms the lowest level of the preprocessor---grouping # text into a line-by-line format. # ---------------------------------------------------------------------- def group_lines(self,input,abssource): r"""Given an input string, this function splits it into lines. Trailing whitespace is removed. This function forms the lowest level of the preprocessor---grouping text into a line-by-line format. """ lex = self.lexer.clone() lines = [x.rstrip() for x in input.splitlines()] input = "\n".join(lines) lex.input(input) lex.lineno = 1 current_line = [] while True: tok = lex.token() if not tok: break tok.source = abssource current_line.append(tok) if tok.type in self.t_WS and tok.value == '\n': yield current_line current_line = [] if current_line: nltok = copy.copy(current_line[-1]) nltok.type = self.t_NEWLINE nltok.value = '\n' current_line.append(nltok) yield current_line # ---------------------------------------------------------------------- # tokenstrip() # # Remove leading/trailing whitespace tokens from a token list # ---------------------------------------------------------------------- def tokenstrip(self,tokens): """Remove leading/trailing whitespace tokens from a token list""" i = 0 while i < len(tokens) and tokens[i].type in self.t_WS: i += 1 del tokens[:i] i = len(tokens)-1 while i >= 0 and tokens[i].type in self.t_WS: i -= 1 del tokens[i+1:] return tokens # ---------------------------------------------------------------------- # collect_args() # # Collects comma separated arguments from a list of tokens. The arguments # must be enclosed in parenthesis. Returns a tuple (tokencount,args,positions) # where tokencount is the number of tokens consumed, args is a list of arguments, # and positions is a list of integers containing the starting index of each # argument. Each argument is represented by a list of tokens. # # When collecting arguments, leading and trailing whitespace is removed # from each argument. # # This function properly handles nested parenthesis and commas---these do not # define new arguments. # ---------------------------------------------------------------------- def collect_args(self,tokenlist,ignore_errors=False): """Collects comma separated arguments from a list of tokens. The arguments must be enclosed in parenthesis. Returns a tuple (tokencount,args,positions) where tokencount is the number of tokens consumed, args is a list of arguments, and positions is a list of integers containing the starting index of each argument. Each argument is represented by a list of tokens. When collecting arguments, leading and trailing whitespace is removed from each argument. This function properly handles nested parenthesis and commas---these do not define new arguments.""" args = [] positions = [] current_arg = [] nesting = 1 tokenlen = len(tokenlist) # Search for the opening '('. i = 0 while (i < tokenlen) and (tokenlist[i].type in self.t_WS): i += 1 if (i < tokenlen) and (tokenlist[i].value == '('): positions.append(i+1) else: if not ignore_errors: self.on_error(tokenlist[0].source,tokenlist[0].lineno,"Missing '(' in macro arguments") return 0, [], [] i += 1 while i < tokenlen: t = tokenlist[i] if t.value == '(': current_arg.append(t) nesting += 1 elif t.value == ')': nesting -= 1 if nesting == 0: args.append(self.tokenstrip(current_arg)) positions.append(i) return i+1,args,positions current_arg.append(t) elif t.value == ',' and nesting == 1: args.append(self.tokenstrip(current_arg)) positions.append(i+1) current_arg = [] else: current_arg.append(t) i += 1 # Missing end argument if not ignore_errors: self.on_error(tokenlist[-1].source,tokenlist[-1].lineno,"Missing ')' in macro arguments") return 0, [],[] # ---------------------------------------------------------------------- # macro_prescan() # # Examine the macro value (token sequence) and identify patch points # This is used to speed up macro expansion later on---we'll know # right away where to apply patches to the value to form the expansion # ---------------------------------------------------------------------- def macro_prescan(self,macro): """Examine the macro value (token sequence) and identify patch points This is used to speed up macro expansion later on---we'll know right away where to apply patches to the value to form the expansion""" macro.patch = [] # Standard macro arguments macro.str_patch = [] # String conversion expansion macro.var_comma_patch = [] # Variadic macro comma patch i = 0 #print("BEFORE", macro.value) #print("BEFORE", [x.value for x in macro.value]) while i < len(macro.value): if macro.value[i].type == self.t_ID and macro.value[i].value in macro.arglist: argnum = macro.arglist.index(macro.value[i].value) # Conversion of argument to a string j = i - 1 while j >= 0 and macro.value[j].type in self.t_WS: j -= 1 if j >= 0 and macro.value[j].value == '#': macro.value[i] = copy.copy(macro.value[i]) macro.value[i].type = self.t_STRING while i > j: del macro.value[j] i -= 1 macro.str_patch.append((argnum,i)) continue # Concatenation elif (i > 0 and macro.value[i-1].value == '##'): macro.patch.append(('t',argnum,i)) i += 1 continue elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'): macro.patch.append(('t',argnum,i)) i += 1 continue # Standard expansion else: macro.patch.append(('e',argnum,i)) elif macro.value[i].value == '##': if macro.variadic and (i > 0) and (macro.value[i-1].value == ',') and \ ((i+1) < len(macro.value)) and (macro.value[i+1].type == self.t_ID) and \ (macro.value[i+1].value == macro.vararg): macro.var_comma_patch.append(i-1) i += 1 macro.patch.sort(key=lambda x: x[2],reverse=True) #print("AFTER", macro.value) #print("AFTER", [x.value for x in macro.value]) #print(macro.patch) # ---------------------------------------------------------------------- # macro_expand_args() # # Given a Macro and list of arguments (each a token list), this method # returns an expanded version of a macro. The return value is a token sequence # representing the replacement macro tokens # ---------------------------------------------------------------------- def macro_expand_args(self,macro,args,expanding_from): """Given a Macro and list of arguments (each a token list), this method returns an expanded version of a macro. The return value is a token sequence representing the replacement macro tokens""" # Make a copy of the macro token sequence rep = [copy.copy(_x) for _x in macro.value] # Make string expansion patches. These do not alter the length of the replacement sequence str_expansion = {} for argnum, i in macro.str_patch: if argnum not in str_expansion: # Strip all non-space whitespace before stringization tokens = copy.copy(args[argnum]) for j in range(len(tokens)): if tokens[j].type in self.t_WS and tokens[j].type != self.t_LINECONT: tokens[j].value = ' ' # Collapse all multiple whitespace too j = 0 while j < len(tokens) - 1: if tokens[j].type in self.t_WS and tokens[j+1].type in self.t_WS: del tokens[j+1] else: j += 1 str = "".join([x.value for x in tokens]) str = str.replace("\\","\\\\").replace('"', '\\"') str_expansion[argnum] = '"' + str + '"' rep[i] = copy.copy(rep[i]) rep[i].value = str_expansion[argnum] # Make the variadic macro comma patch. If the variadic macro argument is empty, we get rid comma_patch = False if macro.variadic and not args[-1]: for i in macro.var_comma_patch: rep[i] = None comma_patch = True # Make all other patches. The order of these matters. It is assumed that the patch list # has been sorted in reverse order of patch location since replacements will cause the # size of the replacement sequence to expand from the patch point. expanded = { } #print("***", macro) #print(macro.patch) for ptype, argnum, i in macro.patch: #print([x.value for x in rep]) # Concatenation. Argument is left unexpanded if ptype == 't': rep[i:i+1] = args[argnum] # Normal expansion. Argument is macro expanded first elif ptype == 'e': #print('*** Function macro arg', rep[i], 'replace with', args[argnum], 'which expands into', self.expand_macros(copy.copy(args[argnum]))) if argnum not in expanded: expanded[argnum] = self.expand_macros(copy.copy(args[argnum]), expanding_from) rep[i:i+1] = expanded[argnum] # Get rid of removed comma if necessary if comma_patch: rep = [_i for _i in rep if _i] # Do a token concatenation pass, stitching any tokens separated by ## into a single token while len(rep) and rep[0].type == self.t_DPOUND: del rep[0] while len(rep) and rep[-1].type == self.t_DPOUND: del rep[-1] i = 1 stitched = False while i < len(rep) - 1: if rep[i].type == self.t_DPOUND: j = i + 1 while rep[j].type == self.t_DPOUND: j += 1 rep[i-1] = copy.copy(rep[i-1]) rep[i-1].type = None rep[i-1].value += rep[j].value while j >= i: del rep[i] j -= 1 stitched = True else: i += 1 if stitched: # Stitched tokens will have unknown type, so figure those out now i = 0 lex = self.lexer.clone() while i < len(rep): if rep[i].type is None: lex.input(rep[i].value) toks = [] while True: tok = lex.token() if not tok: break toks.append(tok) if len(toks) != 1: # Split it once again while len(toks) > 1: rep.insert(i+1, copy.copy(rep[i])) rep[i+1].value = toks[-1].value rep[i+1].type = toks[-1].type toks.pop() rep[i].value = toks[0].value rep[i].type = toks[0].type else: rep[i].type = toks[0].type i += 1 #print rep return rep # ---------------------------------------------------------------------- # expand_macros() # # Given a list of tokens, this function performs macro expansion. # ---------------------------------------------------------------------- def expand_macros(self,tokens,expanding_from=[]): """Given a list of tokens, this function performs macro expansion.""" # Each token needs to track from which macros it has been expanded from to prevent recursion for tok in tokens: if not hasattr(tok, 'expanded_from'): tok.expanded_from = [] i = 0 #print("*** EXPAND MACROS in", "".join([t.value for t in tokens]), "expanding_from=", expanding_from) #print(tokens) #print([(t.value, t.expanded_from) for t in tokens]) while i < len(tokens): t = tokens[i] if self.linemacrodepth == 0: self.linemacro = t.lineno self.linemacrodepth = self.linemacrodepth + 1 if t.type == self.t_ID: if t.value in self.macros and t.value not in t.expanded_from and t.value not in expanding_from: # Yes, we found a macro match m = self.macros[t.value] if m.arglist is None: # A simple macro rep = [copy.copy(_x) for _x in m.value] ex = self.expand_macros(rep, expanding_from + [t.value]) #print("\nExpanding macro", m, "\ninto", ex, "\nreplacing", tokens[i:i+1]) for e in ex: e.source = t.source e.lineno = t.lineno if not hasattr(e, 'expanded_from'): e.expanded_from = [] e.expanded_from.append(t.value) tokens[i:i+1] = ex else: # A macro with arguments j = i + 1 while j < len(tokens) and (tokens[j].type in self.t_WS or tokens[j].type in self.t_COMMENT): j += 1 # A function like macro without an invocation list is to be ignored if j == len(tokens) or tokens[j].value != '(': i = j else: tokcount,args,positions = self.collect_args(tokens[j:], True) if tokcount == 0: # Unclosed parameter list, just bail out break if (not m.variadic # A no arg or single arg consuming macro is permitted to be expanded with nothing and (args != [[]] or len(m.arglist) > 1) and len(args) != len(m.arglist)): self.on_error(t.source,t.lineno,"Macro %s requires %d arguments but was passed %d" % (t.value,len(m.arglist),len(args))) i = j + tokcount elif m.variadic and len(args) < len(m.arglist)-1: if len(m.arglist) > 2: self.on_error(t.source,t.lineno,"Macro %s must have at least %d arguments" % (t.value, len(m.arglist)-1)) else: self.on_error(t.source,t.lineno,"Macro %s must have at least %d argument" % (t.value, len(m.arglist)-1)) i = j + tokcount else: if m.variadic: if len(args) == len(m.arglist)-1: args.append([]) else: args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1] del args[len(m.arglist):] else: # If we called a single arg macro with empty, fake extend args while len(args) < len(m.arglist): args.append([]) # Get macro replacement text rep = self.macro_expand_args(m, args, expanding_from) ex = self.expand_macros(rep, expanding_from + [t.value]) for e in ex: e.source = t.source e.lineno = t.lineno if not hasattr(e, 'expanded_from'): e.expanded_from = [] e.expanded_from.append(t.value) # A non-conforming extension implemented by the GCC and clang preprocessors # is that an expansion of a macro with arguments where the following token is # an identifier inserts a space between the expansion and the identifier. This # differs from Boost.Wave incidentally (see https://github.com/ned14/pcpp/issues/29) if len(tokens) > j+tokcount and tokens[j+tokcount].type in self.t_ID: #print("*** token after expansion is", tokens[j+tokcount]) newtok = copy.copy(tokens[j+tokcount]) newtok.type = self.t_SPACE newtok.value = ' ' ex.append(newtok) #print("\nExpanding macro", m, "\n\ninto", ex, "\n\nreplacing", tokens[i:j+tokcount]) tokens[i:j+tokcount] = ex self.linemacrodepth = self.linemacrodepth - 1 if self.linemacrodepth == 0: self.linemacro = 0 continue elif self.expand_linemacro and t.value == '__LINE__': t.type = self.t_INTEGER t.value = self.t_INTEGER_TYPE(self.linemacro) elif self.expand_countermacro and t.value == '__COUNTER__': t.type = self.t_INTEGER t.value = self.t_INTEGER_TYPE(self.countermacro) self.countermacro += 1 i += 1 self.linemacrodepth = self.linemacrodepth - 1 if self.linemacrodepth == 0: self.linemacro = 0 return tokens # ---------------------------------------------------------------------- # evalexpr() # # Evaluate an expression token sequence for the purposes of evaluating # integral expressions. # ---------------------------------------------------------------------- def evalexpr(self,tokens): """Evaluate an expression token sequence for the purposes of evaluating integral expressions.""" if not tokens: self.on_error('unknown', 0, "Empty expression") return (0, None) # tokens = tokenize(line) # Search for defined macros partial_expansion = False def replace_defined(tokens): i = 0 while i < len(tokens): if tokens[i].type == self.t_ID and tokens[i].value == 'defined': j = i + 1 needparen = False result = "0L" while j < len(tokens): if tokens[j].type in self.t_WS: j += 1 continue elif tokens[j].type == self.t_ID: if tokens[j].value in self.macros: result = "1L" elif not self.passthru_expr_has_include and tokens[j].value == '__has_include': result = "1L" else: repl = self.on_unknown_macro_in_defined_expr(tokens[j]) if repl is None: partial_expansion = True result = 'defined('+tokens[j].value+')' else: result = "1L" if repl else "0L" if not needparen: break elif tokens[j].value == '(': needparen = True elif tokens[j].value == ')': break else: self.on_error(tokens[i].source,tokens[i].lineno,"Malformed defined()") j += 1 if result.startswith('defined'): tokens[i].type = self.t_ID tokens[i].value = result else: tokens[i].type = self.t_INTEGER tokens[i].value = self.t_INTEGER_TYPE(result) del tokens[i+1:j+1] i += 1 return tokens # Replace any defined(macro) before macro expansion tokens = replace_defined(tokens) tokens = self.expand_macros(tokens) # Replace any defined(macro) after macro expansion tokens = replace_defined(tokens) if not self.passthru_expr_has_include: # We need to specially handle _has_include(<...>) because the inner <...> parses as an invalid expression. # We do this by injecting it as a string, and we undo that later. def replace_has_include(tokens): i = 0 while i < len(tokens): if tokens[i].type == self.t_ID and tokens[i].value == '__has_include': j = i + 1 needparen = False bracketpos = -1 while j < len(tokens): if tokens[j].type in self.t_WS: j += 1 continue elif tokens[j].type == self.t_ID: assert bracketpos >= 0 # Convert the into a string tokens[j].type = self.t_STRING tokens[j].value = '"' + ''.join([tokens[x].value for x in range(bracketpos, j + 1)]) del tokens[bracketpos:j] j = bracketpos elif tokens[j].value == '<': bracketpos = j elif tokens[j].value == '>': assert bracketpos > 0 tokens[bracketpos].value += ''.join([tokens[x].value for x in range(bracketpos + 1, j + 1)]) + '"' del tokens[bracketpos + 1:j + 1] j = bracketpos bracketpos = -1 elif tokens[j].value == '(': needparen = True elif tokens[j].value == ')': break elif tokens[j].type != self.t_STRING: self.on_error(tokens[i].source,tokens[i].lineno,"Malformed __has_include()") j += 1 i += 1 return tokens tokens = replace_has_include(tokens) if not tokens: return (0, None) class IndirectToMacroHook(object): def __init__(self, p): self.__preprocessor = p self.partial_expansion = False def __contains__(self, key): return True def __getitem__(self, key): if key.startswith('defined('): self.partial_expansion = True return 0 repl = self.__preprocessor.on_unknown_macro_in_expr(key) #print("*** IndirectToMacroHook[", key, "] returns", repl, file = sys.stderr) if repl is None: self.partial_expansion = True return key return repl evalvars = IndirectToMacroHook(self) class IndirectToHasInclude(object): def __init__(self, p): self.__preprocessor = p def __call__(self, x): #print("*** has_include", x, file = sys.stderr) if x.startswith('"<') and x.endswith('>"'): # Undo our special handling from earlier x = x[1:-1] x = self.__preprocessor.tokenize(x) exists = [ p for p in self.__preprocessor.include(x, x, include_exists_only=True) ] return 1 if exists[0] else 0 class IndirectToMacroFunctionHook(object): def __init__(self, p): self.__preprocessor = p self.partial_expansion = False def __contains__(self, key): return True def __getitem__(self, key): if not self.__preprocessor.passthru_expr_has_include and key == '__has_include': return IndirectToHasInclude(self.__preprocessor) repl = self.__preprocessor.on_unknown_macro_function_in_expr(key) #print("*** IndirectToMacroFunctionHook[", key, "] returns", repl, file = sys.stderr) if repl is None: self.partial_expansion = True return key return repl evalfuncts = IndirectToMacroFunctionHook(self) try: result = self.evaluator(tokens, functions = evalfuncts, identifiers = evalvars).value() partial_expansion = partial_expansion or evalvars.partial_expansion or evalfuncts.partial_expansion except OutputDirective: raise except Exception as e: partial_expansion = partial_expansion or evalvars.partial_expansion or evalfuncts.partial_expansion if not partial_expansion: self.on_error(tokens[0].source,tokens[0].lineno,"Could not evaluate expression due to %s (passed to evaluator: '%s')" % (repr(e), ''.join([tok.value for tok in tokens]))) result = 0 return (result, tokens) if partial_expansion else (result, None) # ---------------------------------------------------------------------- # parsegen() # # Parse an input string # ---------------------------------------------------------------------- def parsegen(self,input,source=None,abssource=None): """Parse an input string""" rewritten_source = source if abssource: rewritten_source = abssource for rewrite in self.rewrite_paths: temp = re.sub(rewrite[0], rewrite[1], rewritten_source) if temp != abssource: rewritten_source = temp if os.sep != '/': rewritten_source = rewritten_source.replace(os.sep, '/') break # Replace trigraph sequences t = trigraph(input) if self.enable_trigraphs else input lines = self.group_lines(t, rewritten_source) if not source: source = "" if not rewritten_source: rewritten_source = "" my_include_times_idx = len(self.include_times) self.include_times.append(FileInclusionTime(self.macros['__FILE__'] if '__FILE__' in self.macros else None, source, abssource, self.include_depth)) self.include_depth += 1 my_include_time_begin = clock() if self.expand_filemacro: self.define("__FILE__ \"%s\"" % rewritten_source) self.source = abssource chunk = [] enable = True iftrigger = False ifpassthru = False class ifstackentry(object): def __init__(self,enable,iftrigger,ifpassthru,startlinetoks): self.enable = enable self.iftrigger = iftrigger self.ifpassthru = ifpassthru self.rewritten = False self.startlinetoks = startlinetoks ifstack = [] # True until any non-whitespace output or anything with effects happens. at_front_of_file = True # True if auto pragma once still a possibility for this #include auto_pragma_once_possible = self.auto_pragma_once_enabled # =(MACRO, 0) means #ifndef MACRO or #if !defined(MACRO) seen, =(MACRO,1) means #define MACRO seen include_guard = None self.on_potential_include_guard(None) for x in lines: all_whitespace = True skip_auto_pragma_once_possible_check = False # Handle comments for i,tok in enumerate(x): if tok.type in self.t_COMMENT: if not self.on_comment(tok): if tok.type == self.t_COMMENT1: tok.value = ' ' elif tok.type == self.t_COMMENT2: tok.value = '\n' tok.type = 'CPP_WS' # Skip over whitespace for i,tok in enumerate(x): if tok.type not in self.t_WS and tok.type not in self.t_COMMENT: all_whitespace = False break output_and_expand_line = True output_unexpanded_line = False if tok.value == '#': precedingtoks = [ tok ] output_and_expand_line = False try: # Preprocessor directive i += 1 while i < len(x) and x[i].type in self.t_WS: precedingtoks.append(x[i]) i += 1 dirtokens = self.tokenstrip(x[i:]) if dirtokens: name = dirtokens[0].value args = self.tokenstrip(dirtokens[1:]) if self.debugout is not None: print("%d:%d:%d %s:%d #%s %s" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno, dirtokens[0].value, "".join([tok.value for tok in args])), file = self.debugout) #print(ifstack) handling = self.on_directive_handle(dirtokens[0],args,ifpassthru,precedingtoks) assert handling == True or handling == None else: name = "" args = [] raise OutputDirective(Action.IgnoreAndRemove) if name == 'define': at_front_of_file = False if enable: for tok in self.expand_macros(chunk): yield tok chunk = [] if include_guard and include_guard[1] == 0: if include_guard[0] == args[0].value and len(args) == 1: include_guard = (args[0].value, 1) # If ifpassthru is only turned on due to this include guard, turn it off if ifpassthru and not ifstack[-1].ifpassthru: ifpassthru = False self.define(args) if self.debugout is not None: print("%d:%d:%d %s:%d %s" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno, repr(self.macros[args[0].value])), file = self.debugout) if handling is None: for tok in x: yield tok elif name == 'include' or (self.include_next_enabled and name == 'include_next'): if enable: for tok in self.expand_macros(chunk): yield tok chunk = [] oldfile = self.macros['__FILE__'] if '__FILE__' in self.macros else None if args and args[0].value != '<' and args[0].type != self.t_STRING: args = self.tokenstrip(self.expand_macros(args)) # print('***', ''.join([x.value for x in args]), file = sys.stderr) for tok in self.include(args, x, include_next_is_active = (name == 'include_next' and abssource is not None)): yield tok if oldfile is not None: self.macros['__FILE__'] = oldfile self.source = abssource elif name == 'undef': at_front_of_file = False if enable: for tok in self.expand_macros(chunk): yield tok chunk = [] self.undef(args) if handling is None: for tok in x: yield tok elif name == 'ifdef': at_front_of_file = False ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x)) if enable: ifpassthru = False if not args[0].value in self.macros and (self.passthru_expr_has_include or args[0].value != '__has_include'): res = self.on_unknown_macro_in_defined_expr(args[0]) if res is None: ifpassthru = True ifstack[-1].rewritten = True raise OutputDirective(Action.IgnoreAndPassThrough) elif res is True: iftrigger = True else: enable = False iftrigger = False else: iftrigger = True elif name == 'ifndef': if not ifstack and at_front_of_file: self.on_potential_include_guard(args[0].value) include_guard = (args[0].value, 0) at_front_of_file = False ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x)) if enable: ifpassthru = False if args[0].value in self.macros or (not self.passthru_expr_has_include and args[0].value == '__has_include'): enable = False iftrigger = False else: res = self.on_unknown_macro_in_defined_expr(args[0]) if res is None: ifpassthru = True ifstack[-1].rewritten = True raise OutputDirective(Action.IgnoreAndPassThrough) elif res is True: enable = False iftrigger = False else: iftrigger = True elif name == 'if': if not ifstack and at_front_of_file: if args[0].value == '!' and args[1].value == 'defined': n = 2 if args[n].value == '(': n += 1 self.on_potential_include_guard(args[n].value) include_guard = (args[n].value, 0) at_front_of_file = False ifstack.append(ifstackentry(enable,iftrigger,ifpassthru,x)) if enable: iftrigger = False ifpassthru = False result, rewritten = self.evalexpr(args) if rewritten is not None: x = x[:i+2] + rewritten + [x[-1]] x[i+1] = copy.copy(x[i+1]) x[i+1].type = self.t_SPACE x[i+1].value = ' ' ifpassthru = True ifstack[-1].rewritten = True raise OutputDirective(Action.IgnoreAndPassThrough) if not result: enable = False else: iftrigger = True elif name == 'elif': at_front_of_file = False if ifstack: if ifstack[-1].enable: # We only pay attention if outer "if" allows this if enable and not ifpassthru: # If already true, we flip enable False enable = False elif not iftrigger: # If False, but not triggered yet, we'll check expression result, rewritten = self.evalexpr(args) if rewritten is not None: enable = True if not ifpassthru: # This is a passthru #elif after a False #if, so convert to an #if x[i].value = 'if' x = x[:i+2] + rewritten + [x[-1]] x[i+1] = copy.copy(x[i+1]) x[i+1].type = self.t_SPACE x[i+1].value = ' ' ifpassthru = True ifstack[-1].rewritten = True raise OutputDirective(Action.IgnoreAndPassThrough) if ifpassthru: # If this elif can only ever be true, simulate that if result: newtok = copy.copy(x[i+3]) newtok.type = self.t_INTEGER newtok.value = self.t_INTEGER_TYPE(result) x = x[:i+2] + [newtok] + [x[-1]] raise OutputDirective(Action.IgnoreAndPassThrough) # Otherwise elide enable = False elif result: enable = True iftrigger = True else: self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #elif") elif name == 'else': at_front_of_file = False if ifstack: if ifstack[-1].enable: if ifpassthru: enable = True raise OutputDirective(Action.IgnoreAndPassThrough) if enable: enable = False elif not iftrigger: enable = True iftrigger = True else: self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #else") elif name == 'endif': at_front_of_file = False if ifstack: oldifstackentry = ifstack.pop() enable = oldifstackentry.enable iftrigger = oldifstackentry.iftrigger ifpassthru = oldifstackentry.ifpassthru if self.debugout is not None: print("%d:%d:%d %s:%d (%s:%d %s)" % (enable, iftrigger, ifpassthru, dirtokens[0].source, dirtokens[0].lineno, oldifstackentry.startlinetoks[0].source, oldifstackentry.startlinetoks[0].lineno, "".join([n.value for n in oldifstackentry.startlinetoks])), file = self.debugout) skip_auto_pragma_once_possible_check = True if oldifstackentry.rewritten: raise OutputDirective(Action.IgnoreAndPassThrough) else: self.on_error(dirtokens[0].source,dirtokens[0].lineno,"Misplaced #endif") elif name == 'pragma' and args[0].value == 'once': if enable: self.include_once[self.source] = None elif enable: # Unknown preprocessor directive output_unexpanded_line = (self.on_directive_unknown(dirtokens[0], args, ifpassthru, precedingtoks) is None) except OutputDirective as e: if e.action == Action.IgnoreAndPassThrough: output_unexpanded_line = True elif e.action == Action.IgnoreAndRemove: pass else: assert False # If there is ever any non-whitespace output outside an include guard, auto pragma once is not possible if not skip_auto_pragma_once_possible_check and auto_pragma_once_possible and not ifstack and not all_whitespace: auto_pragma_once_possible = False if self.debugout is not None: print("%d:%d:%d %s:%d Determined that #include \"%s\" is not entirely wrapped in an include guard macro, disabling auto-applying #pragma once" % (enable, iftrigger, ifpassthru, x[0].source, x[0].lineno, self.source), file = self.debugout) if output_and_expand_line or output_unexpanded_line: if not all_whitespace: at_front_of_file = False # Normal text if enable: if output_and_expand_line: chunk.extend(x) elif output_unexpanded_line: for tok in self.expand_macros(chunk): yield tok chunk = [] for tok in x: yield tok else: # Need to extend with the same number of blank lines i = 0 while i < len(x): if x[i].type not in self.t_WS: del x[i] else: i += 1 chunk.extend(x) for tok in self.expand_macros(chunk): yield tok chunk = [] for i in ifstack: self.on_error(i.startlinetoks[0].source, i.startlinetoks[0].lineno, "Unterminated " + "".join([n.value for n in i.startlinetoks])) if auto_pragma_once_possible and include_guard and include_guard[1] == 1: if self.debugout is not None: print("%d:%d:%d %s:%d Determined that #include \"%s\" is entirely wrapped in an include guard macro called %s, auto-applying #pragma once" % (enable, iftrigger, ifpassthru, self.source, 0, self.source, include_guard[0]), file = self.debugout) self.include_once[self.source] = include_guard[0] elif self.auto_pragma_once_enabled and self.source not in self.include_once: if self.debugout is not None: print("%d:%d:%d %s:%d Did not auto apply #pragma once to this file due to auto_pragma_once_possible=%d, include_guard=%s" % (enable, iftrigger, ifpassthru, self.source, 0, auto_pragma_once_possible, repr(include_guard)), file = self.debugout) my_include_time_end = clock() self.include_times[my_include_times_idx].elapsed = my_include_time_end - my_include_time_begin self.include_depth -= 1 # ---------------------------------------------------------------------- # include() # # Implementation of file-inclusion # ---------------------------------------------------------------------- def include(self,tokens,original_line,include_next_is_active=False,include_exists_only=False): """Implementation of file-inclusion""" # Try to extract the filename and then process an include file if not tokens: return if tokens: if tokens[0].value != '<' and tokens[0].type != self.t_STRING: tokens = self.tokenstrip(self.expand_macros(tokens)) is_system_include = False if tokens[0].value == '<': is_system_include = True # Include <...> i = 1 while i < len(tokens): if tokens[i].value == '>': break i += 1 else: self.on_error(tokens[0].source,tokens[0].lineno,"Malformed #include <...>") return filename = "".join([x.value for x in tokens[1:i]]) if not include_next_is_active: # Search only formally specified paths path = self.path else: # include_next triggered this, must not differentiate path = self.temp_path + self.path elif tokens[0].type == self.t_STRING: filename = tokens[0].value[1:-1] # Search from each nested include file, as well as formally specified paths path = self.temp_path + self.path else: p = self.on_include_not_found(True,False,self.temp_path[0] if self.temp_path else '',tokens[0].value) assert p is None return if not path: path = [''] while True: #print path for p in path: iname = os.path.join(p,filename) fulliname = os.path.abspath(iname) if not include_exists_only and fulliname in self.include_once: if self.debugout is not None: print("x:x:x x:x #include \"%s\" skipped as already seen" % (fulliname), file = self.debugout) if self.passthru_includes is not None and self.passthru_includes.match(''.join([x.value for x in tokens])): for tok in original_line: yield tok return try: ih = self.on_file_open(is_system_include,fulliname) if include_exists_only: ih.close() yield True return unique_id = self.__file_unique_id(ih) if include_next_is_active and unique_id in self.current_include_next_unique_ids: ih.close() continue data = ih.read() ih.close() dname = os.path.dirname(fulliname) if dname: self.temp_path.insert(0,dname) self.current_include_next_unique_ids.append(unique_id) if self.passthru_includes is not None and self.passthru_includes.match(''.join([x.value for x in tokens])): for tok in original_line: yield tok for tok in self.parsegen(data,filename,fulliname): pass else: for tok in self.parsegen(data,filename,fulliname): yield tok self.current_include_next_unique_ids.remove(unique_id) if dname: del self.temp_path[0] return except IOError: pass else: if include_exists_only: yield False return p = self.on_include_not_found(False,is_system_include,self.temp_path[0] if self.temp_path else '',filename) assert p is not None path.append(p) # ---------------------------------------------------------------------- # define() # # Define a new macro # ---------------------------------------------------------------------- def define(self,tokens): """Define a new macro""" if isinstance(tokens,STRING_TYPES): tokens = self.tokenize(tokens) else: tokens = [copy.copy(tok) for tok in tokens] def add_macro(self, name, macro): macro.source = name.source macro.lineno = name.lineno self.macros[name.value] = macro linetok = tokens try: name = linetok[0] if len(linetok) > 1: mtype = linetok[1] else: mtype = None if not mtype: m = Macro(name.value,[]) add_macro(self, name, m) elif mtype.type in self.t_WS: # A normal macro m = Macro(name.value,self.tokenstrip(linetok[2:])) add_macro(self, name, m) elif mtype.value == '(': # A macro with arguments tokcount, args, positions = self.collect_args(linetok[1:]) variadic = False for a in args: if variadic: self.on_error(name.source,name.lineno,"No more arguments may follow a variadic argument") break astr = "".join([str(_i.value) for _i in a]) if astr == "...": variadic = True a[0].type = self.t_ID a[0].value = '__VA_ARGS__' variadic = True del a[1:] continue elif astr[-3:] == "..." and a[0].type == self.t_ID: variadic = True del a[1:] # If, for some reason, "." is part of the identifier, strip off the name for the purposes # of macro expansion if a[0].value[-3:] == '...': a[0].value = a[0].value[:-3] continue # Empty arguments are permitted if len(a) == 0 and len(args) == 1: continue if len(a) > 1 or a[0].type != self.t_ID: self.on_error(a[0].source,a[0].lineno,"Invalid macro argument") break else: mvalue = self.tokenstrip(linetok[1+tokcount:]) i = 0 while i < len(mvalue): if i+1 < len(mvalue): if mvalue[i].type in self.t_WS and mvalue[i+1].value == '##': del mvalue[i] continue elif mvalue[i].value == '##' and mvalue[i+1].type in self.t_WS: del mvalue[i+1] i += 1 m = Macro(name.value,mvalue,[x[0].value for x in args] if args != [[]] else [],variadic) self.macro_prescan(m) add_macro(self, name, m) else: self.on_error(name.source,name.lineno,"Bad macro definition") #except LookupError: # print("Bad macro definition") except: raise # ---------------------------------------------------------------------- # undef() # # Undefine a macro # ---------------------------------------------------------------------- def undef(self,tokens): """Undefine a macro""" if isinstance(tokens,STRING_TYPES): tokens = self.tokenize(tokens) id = tokens[0].value try: del self.macros[id] except LookupError: pass # ---------------------------------------------------------------------- # parse() # # Parse input text. # ---------------------------------------------------------------------- def parse(self,input,source=None,ignore={}): """Parse input text.""" if isinstance(input, FILE_TYPES): if source is None: source = input.name input = input.read() self.ignore = ignore self.parser = self.parsegen(input,source,os.path.abspath(source) if source else None) if source is not None: dname = os.path.dirname(source) self.temp_path.insert(0,dname) # ---------------------------------------------------------------------- # token() # # Method to return individual tokens # ---------------------------------------------------------------------- def token(self): """Method to return individual tokens""" try: while True: tok = next(self.parser) if tok.type not in self.ignore: return tok except StopIteration: self.parser = None return None def write(self, oh=sys.stdout): """Calls token() repeatedly, expanding tokens to their text and writing to the file like stream oh""" lastlineno = 0 lastsource = None done = False blanklines = 0 while not done: emitlinedirective = False toks = [] all_ws = True # Accumulate a line while not done: tok = self.token() if not tok: done = True break toks.append(tok) if tok.value and tok.value[0] == '\n': break if tok.type not in self.t_WS: all_ws = False if not toks: break if all_ws: # Remove preceding whitespace so it becomes just a LF if len(toks) > 1: tok = toks[-1] toks = [ tok ] blanklines += toks[0].value.count('\n') continue # Filter out line continuations, collapsing before and after if needs be for n in range(len(toks)-1, -1, -1): if toks[n].type in self.t_LINECONT: if n > 0 and n < len(toks) - 2 and toks[n-1].type in self.t_WS and toks[n+1].type in self.t_WS: if toks[n-1].type not in self.t_LINECONT: toks[n-1].value = toks[n-1].value[0] del toks[n:n+2] else: del toks[n] # The line in toks is not all whitespace emitlinedirective = (blanklines > 6) and self.line_directive is not None if hasattr(toks[0], 'source'): if lastsource is None: if toks[0].source is not None: emitlinedirective = True lastsource = toks[0].source elif lastsource != toks[0].source: emitlinedirective = True lastsource = toks[0].source # Replace consecutive whitespace in output with a single space except at any indent first_ws = None #print(toks) for n in range(len(toks)-1, -1, -1): tok = toks[n] if first_ws is None: if tok.type in self.t_SPACE or len(tok.value) == 0: first_ws = n else: if tok.type not in self.t_SPACE and len(tok.value) > 0: m = n + 1 while m != first_ws: del toks[m] first_ws -= 1 first_ws = None if self.compress > 0: # Collapse a token of many whitespace into single if toks[m].value and toks[m].value[0] == ' ': toks[m].value = ' ' if not self.compress > 1 and not emitlinedirective: newlinesneeded = toks[0].lineno - lastlineno - 1 if newlinesneeded > 6 and self.line_directive is not None: emitlinedirective = True else: while newlinesneeded > 0: oh.write('\n') newlinesneeded -= 1 lastlineno = toks[0].lineno # Account for those newlines in a multiline comment if emitlinedirective and self.line_directive is not None: oh.write(self.line_directive + ' ' + str(lastlineno) + ('' if lastsource is None else (' "' + lastsource + '"' )) + '\n') for tok in toks: if tok.type == self.t_COMMENT1: lastlineno += tok.value.count('\n') blanklines = 0 #print toks[0].lineno, for tok in toks: #print tok.value, oh.write(tok.value) #print '' if __name__ == "__main__": import doctest doctest.testmod() ================================================ FILE: pyproject.toml ================================================ [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] name = "pcpp" version = "1.31" description = "A C99 preprocessor written in pure Python" readme = "README.rst" license = {file = "LICENSE.txt"} authors = [ {name = "Niall Douglas and David Beazley"} ] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Topic :: Software Development :: Build Tools", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12" ] requires-python = ">=3.6" [project.scripts] pcpp = "pcpp:main" [tool.setuptools] packages = ["pcpp", "pcpp.ply.ply"] [tool.setuptools.package-data] pcpp = ["../LICENSE.txt"] [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["test_*.py", "*_test.py", "tests/*.py"] addopts = ["-v"] ================================================ FILE: requirements.txt ================================================ setuptools wheel pytest ================================================ FILE: setup.py ================================================ #!/usr/bin/env python from setuptools import setup from setuptools.command.test import test as TestCommand import os import re import sys class PyTest(TestCommand): """Custom test command that uses pytest""" user_options = [('pytest-args=', 'a', "Arguments to pass to pytest")] def initialize_options(self): TestCommand.initialize_options(self) self.pytest_args = [] def run_tests(self): import subprocess import sys # Install pytest if not available try: import pytest except ImportError: print("Installing pytest...") subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'pytest']) import pytest # Run pytest errno = pytest.main(['tests/'] + self.pytest_args) sys.exit(errno) here = os.path.abspath(os.path.dirname(__file__)) # Get the long description from the README file with open(os.path.join(here, 'README.rst')) as f: long_description = f.read() # Read version from pcpp/pcmd.py without importing it with open(os.path.join(here, 'pcpp', 'pcmd.py')) as f: content = f.read() version_match = re.search(r"^version=['\"]([^'\"]*)['\"]", content, re.M) if version_match: version = version_match.group(1) else: raise RuntimeError("Unable to find version string.") setup( name='pcpp', version=version, description='A C99 preprocessor written in pure Python', long_description=long_description, author='Niall Douglas and David Beazley', url='https://github.com/ned14/pcpp', packages=['pcpp', 'pcpp.ply.ply'], package_data={'pcpp' : ['../LICENSE.txt']}, entry_points={ 'console_scripts': [ 'pcpp=pcpp:main' ] }, options={'bdist_wheel':{'universal':False}}, # Changed to False since we're Python 3 only now license='BSD', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Topic :: Software Development :: Build Tools', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', ], cmdclass={'test': PyTest}, ) ================================================ FILE: tests/Readme.md ================================================ test-c was borrowed from the test suite for the mcpp preprocessor http://mcpp.sourceforge.net/ ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/alternate_input_encodings.py ================================================ import unittest, sys, io, os shouldbe1 = r'''#line 1 "tests/alternate_input_encodings1_ucs16le.c" 语言处理 ''' shouldbe2 = r'''#line 1 "tests/alternate_input_encodings1_ucs16le.c" 语言处理 #line 1 "tests/alternate_input_encodings2_ucs16le.c" いろはにほへとちりぬるを ''' class runner(object): def runTest(self): from pcpp import CmdPreprocessor if self.multiple: p = CmdPreprocessor(['pcpp', '-o', 'tests/alternate_input_encodings.c', '--assume-input-encoding', 'utf_16_le', '--output-encoding', 'utf_8', 'tests/alternate_input_encodings1_ucs16le.c', 'tests/alternate_input_encodings2_ucs16le.c']) else: p = CmdPreprocessor(['pcpp', '-o', 'tests/alternate_input_encodings.c', '--assume-input-encoding', 'utf_16_le', '--output-encoding', 'utf_8', 'tests/alternate_input_encodings1_ucs16le.c']) with io.open('tests/alternate_input_encodings.c', 'rt', encoding='utf-8') as ih: output = ih.read() os.remove('tests/alternate_input_encodings.c') if output != self.shouldbe: print("Should be:\n" + repr(self.shouldbe) + "EOF\n", file = sys.stderr) print("\nWas:\n" + repr(output) + "EOF\n", file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(output, self.shouldbe) class single_input_file(unittest.TestCase, runner): multiple = False shouldbe = shouldbe1 class multiple_input_files(unittest.TestCase, runner): multiple = True shouldbe = shouldbe2 ================================================ FILE: tests/cstd.py ================================================ import unittest, time from io import StringIO clock = time.process_time class runner(object): def runTest(self): from pcpp import Preprocessor import os, sys start = clock() p = Preprocessor() p.parse(self.input) oh = StringIO() p.write(oh) end = clock() print("Preprocessed test in", end-start, "seconds") if oh.getvalue() != self.output: print("Should be:\n" + self.output, file = sys.stderr) print("\n\nWas:\n" + oh.getvalue(), file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(oh.getvalue(), self.output) # These preprocessor test fragments are borrowed from the C11 standard class std1(unittest.TestCase, runner): input = r"""#define x 3 #define f(a) f(x * (a)) #undef x #define x 2 #define g f #define z z[0] #define h g(~ #define m(a) a(w) #define w 0,1 #define t(a) a #define p() int #define q(x) x #define r(x,y) x ## y #define str(x) # x f(y+1) + f(f(z)) % t(t(g)(0) + t)(1); g(x+(3,4)-w) | h 5) & m (f)^m(m); p() i[q()] = { q(1), r(2,3), r(4,), r(,5), r(,) }; char c[2][6] = { str(hello), str() };""" output = r"""#line 15 f(2 * (y+1)) + f(2 * (f(2 * (z[0])))) % f(2 * (0)) + t(1); f(2 * (2+(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))^m(0,1); int i[] = { 1, 23, 4, 5, }; char c[2][6] = { "hello", "" }; """ class std2(unittest.TestCase, runner): input = r"""#define TWO_ARGS a,b #define sub( x, y) (x - y) assert( sub( TWO_ARGS, 1) == 1); """ output = r""" assert( (a,b - 1) == 1); """ class std3(unittest.TestCase, runner): input = r"""#define t(x,y,z) x ## y ## z int j[] = { t(1,2,3), t(,4,5), t(6,,7), t(8,9,), t(10,,), t(,11,), t(,,12), t(,,) }; """ output = r""" int j[] = { 123, 45, 67, 89, 10, 11, 12, }; """ class std4(unittest.TestCase, runner): input = r"""#define debug(...) fprintf(stderr, __VA_ARGS__) #define showlist(...) puts(#__VA_ARGS__) #define report(test, ...) ((test)?puts(#test):\ printf(__VA_ARGS__)) debug("Flag"); debug("X = %d\n", x); showlist(The first, second, and third items.); report(x>y, "x is %d but y is %d", x, y); """ output = r""" fprintf(stderr, "Flag"); fprintf(stderr, "X = %d\n", x); puts("The first, second, and third items."); ((x>y)?puts("x>y"):printf( "x is %d but y is %d", x, y)); """ class std5(unittest.TestCase, runner): input = r"""#define Z Z[0] assert( Z == 1); #define AB BA #define BA AB assert( AB == 1); #define f(a) a + f(a) assert( f( x) == 2); #define g(a) a + h( a) #define h(a) a + g( a) assert( g( x) == 4); assert( f( Z) == 2); """ output = r""" assert( Z[0] == 1); assert( AB == 1); assert( x + f(x) == 2); assert( x + x + g( x) == 4); assert( Z[0] + f(Z[0]) == 2); """ class std6(unittest.TestCase, runner): input = r"""#define MACRO_0 0 #define MACRO_1 1 #define glue( a, b) a ## b assert( glue( MACRO_0, MACRO_1) == 2); """ output = r""" assert( MACRO_0MACRO_1 == 2); """ class std7(unittest.TestCase, runner): input = r"""#if 0 "nonsence"; /* #else still in comment */ #else #define MACRO_abcd /* in comment */ abcd #endif assert( MACRO_abcd == 4); """ output = r"""#line 11 assert( abcd == 4); """ class std8(unittest.TestCase, runner): input = r"""#if 0 niall #elif 0 douglas #elif 1 foo #endif """ output = r""" foo """ class std9(unittest.TestCase, runner): input = r'''#define str(x) # x str( niall is a /* comment */ pretty boy ) ''' output = r''' "niall is a pretty boy" ''' class std10(unittest.TestCase, runner): input = r"""#define MACRO_0 0 #define MACRO_1 1 #define glue( a, b) a ## b assert( glue( MACRO_, 1) == 1); """ output = r""" assert( 1 == 1); """ class std11(unittest.TestCase, runner): input = r"""#define FUNC( a, b, c) a + b + c FUNC ( a, b, c ) == 6 """ output = r""" a + b + c == 6 """ class test12(unittest.TestCase, runner): input = r""" #define BOOSTLITE_GLUE(x, y) x y #define BOOSTLITE_RETURN_ARG_COUNT(_1_, _2_, _3_, _4_, _5_, _6_, _7_, _8_, count, ...) count #define BOOSTLITE_EXPAND_ARGS(args) BOOSTLITE_RETURN_ARG_COUNT args #define BOOSTLITE_COUNT_ARGS_MAX8(...) BOOSTLITE_EXPAND_ARGS((__VA_ARGS__, 8, 7, 6, 5, 4, 3, 2, 1, 0)) #define BOOSTLITE_OVERLOAD_MACRO2(name, count) name##count #define BOOSTLITE_OVERLOAD_MACRO1(name, count) BOOSTLITE_OVERLOAD_MACRO2(name, count) #define BOOSTLITE_OVERLOAD_MACRO(name, count) BOOSTLITE_OVERLOAD_MACRO1(name, count) #define BOOSTLITE_CALL_OVERLOAD(name, ...) BOOSTLITE_GLUE(BOOSTLITE_OVERLOAD_MACRO(name, BOOSTLITE_COUNT_ARGS_MAX8(__VA_ARGS__)), (__VA_ARGS__)) #define BOOSTLITE_GLUE_(x, y) x y #define BOOSTLITE_RETURN_ARG_COUNT_(_1_, _2_, _3_, _4_, _5_, _6_, _7_, _8_, count, ...) count #define BOOSTLITE_EXPAND_ARGS_(args) BOOSTLITE_RETURN_ARG_COUNT_ args #define BOOSTLITE_COUNT_ARGS_MAX8_(...) BOOSTLITE_EXPAND_ARGS_((__VA_ARGS__, 8, 7, 6, 5, 4, 3, 2, 1, 0)) #define BOOSTLITE_OVERLOAD_MACRO2_(name, count) name##count #define BOOSTLITE_OVERLOAD_MACRO1_(name, count) BOOSTLITE_OVERLOAD_MACRO2_(name, count) #define BOOSTLITE_OVERLOAD_MACRO_(name, count) BOOSTLITE_OVERLOAD_MACRO1_(name, count) #define BOOSTLITE_CALL_OVERLOAD_(name, ...) BOOSTLITE_GLUE_(BOOSTLITE_OVERLOAD_MACRO_(name, BOOSTLITE_COUNT_ARGS_MAX8_(__VA_ARGS__)), (__VA_ARGS__)) #define BOOSTLITE_BIND_STRINGIZE(a) #a #define BOOSTLITE_BIND_STRINGIZE2(a) BOOSTLITE_BIND_STRINGIZE(a) #define BOOSTLITE_BIND_NAMESPACE_VERSION8(a, b, c, d, e, f, g, h) a##_##b##_##c##_##d##_##e##_##f##_##g##_##h #define BOOSTLITE_BIND_NAMESPACE_VERSION7(a, b, c, d, e, f, g) a##_##b##_##c##_##d##_##e##_##f##_##g #define BOOSTLITE_BIND_NAMESPACE_VERSION6(a, b, c, d, e, f) a##_##b##_##c##_##d##_##e##_##f #define BOOSTLITE_BIND_NAMESPACE_VERSION5(a, b, c, d, e) a##_##b##_##c##_##d##_##e #define BOOSTLITE_BIND_NAMESPACE_VERSION4(a, b, c, d) a##_##b##_##c##_##d #define BOOSTLITE_BIND_NAMESPACE_VERSION3(a, b, c) a##_##b##_##c #define BOOSTLITE_BIND_NAMESPACE_VERSION2(a, b) a##_##b #define BOOSTLITE_BIND_NAMESPACE_VERSION1(a) a #define BOOSTLITE_BIND_NAMESPACE_VERSION(...) BOOSTLITE_CALL_OVERLOAD(BOOSTLITE_BIND_NAMESPACE_VERSION, __VA_ARGS__) #define BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT2(name, modifier) modifier namespace name { #define BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT1(name) namespace name { #define BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT(...) BOOSTLITE_CALL_OVERLOAD_(BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT, __VA_ARGS__) #define BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND8(a, b, c, d, e, f, g, h) BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT a BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND7(b, c, d, e, f, g, h) #define BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND7(a, b, c, d, e, f, g) BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT a BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND6(b, c, d, e, f, g) #define BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND6(a, b, c, d, e, f) BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT a BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND5(b, c, d, e, f) #define BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND5(a, b, c, d, e) BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT a BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND4(b, c, d, e) #define BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND4(a, b, c, d) BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT a BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND3(b, c, d) #define BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND3(a, b, c) BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT a BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND2(b, c) #define BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND2(a, b) BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT a BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND1(b) #define BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND1(a) BOOSTLITE_BIND_NAMESPACE_BEGIN_NAMESPACE_SELECT a #define BOOSTLITE_BIND_NAMESPACE_BEGIN(...) BOOSTLITE_CALL_OVERLOAD(BOOSTLITE_BIND_NAMESPACE_BEGIN_EXPAND, __VA_ARGS__) #define BOOST_OUTCOME_VERSION_GLUE2(a, b, c) a##b##c #define BOOST_OUTCOME_VERSION_GLUE(a, b, c) BOOST_OUTCOME_VERSION_GLUE2(a, b, c) #define BOOST_OUTCOME_VERSION_MAJOR 1 #define BOOST_OUTCOME_VERSION_MINOR 0 #define BOOST_OUTCOME_VERSION_PATCH 0 #define BOOST_OUTCOME_VERSION_REVISION 0 #define BOOST_OUTCOME_NAMESPACE_VERSION BOOST_OUTCOME_VERSION_GLUE(BOOST_OUTCOME_VERSION_MAJOR, _, BOOST_OUTCOME_VERSION_MINOR) #define BOOST_OUTCOME_PREVIOUS_COMMIT_UNIQUE 01320023 #define BOOST_OUTCOME_V1_STL11_IMPL std #define BOOST_OUTCOME_V1_ERROR_CODE_IMPL std #define BOOST_OUTCOME_V1 (boost), (outcome), (BOOSTLITE_BIND_NAMESPACE_VERSION(, BOOST_OUTCOME_NAMESPACE_VERSION, BOOST_OUTCOME_V1_STL11_IMPL, BOOST_OUTCOME_V1_ERROR_CODE_IMPL, BOOST_OUTCOME_PREVIOUS_COMMIT_UNIQUE), inline) #define BOOST_OUTCOME_V1_NAMESPACE_BEGIN BOOSTLITE_BIND_NAMESPACE_BEGIN(BOOST_OUTCOME_V1) BOOSTLITE_BIND_NAMESPACE_VERSION(, BOOST_OUTCOME_NAMESPACE_VERSION, BOOST_OUTCOME_V1_STL11_IMPL, BOOST_OUTCOME_V1_ERROR_CODE_IMPL, BOOST_OUTCOME_PREVIOUS_COMMIT_UNIQUE) BOOST_OUTCOME_V1_NAMESPACE_BEGIN """ output = r"""#line 67 _1_0_std_std_01320023 namespace boost { namespace outcome { inline namespace _1_0_std_std_01320023 { """ class test13(unittest.TestCase, runner): input = r""" #define _CRT_INTERNAL_NONSTDC_NAMES \ ( \ ( defined _CRT_DECLARE_NONSTDC_NAMES && _CRT_DECLARE_NONSTDC_NAMES) || \ (!defined _CRT_DECLARE_NONSTDC_NAMES && !__STDC__ ) \ ) #if _CRT_INTERNAL_NONSTDC_NAMES foo #endif """ output = r"""#line 8 foo """ class test14(unittest.TestCase, runner): input = r""" # if defined __GNUC__ // NOTE: GNUC is also defined for Clang # if (__GNUC__ == 4) && (__GNUC_MINOR__ >= 8) # define TR2_OPTIONAL_GCC_4_8_AND_HIGHER___ # elif (__GNUC__ > 4) # define TR2_OPTIONAL_GCC_4_8_AND_HIGHER___ # endif # # if (__GNUC__ == 4) && (__GNUC_MINOR__ >= 7) # define TR2_OPTIONAL_GCC_4_7_AND_HIGHER___ # elif (__GNUC__ > 4) # define TR2_OPTIONAL_GCC_4_7_AND_HIGHER___ # endif # # if (__GNUC__ == 4) && (__GNUC_MINOR__ == 8) && (__GNUC_PATCHLEVEL__ >= 1) # define TR2_OPTIONAL_GCC_4_8_1_AND_HIGHER___ # elif (__GNUC__ == 4) && (__GNUC_MINOR__ >= 9) # define TR2_OPTIONAL_GCC_4_8_1_AND_HIGHER___ # elif (__GNUC__ > 4) # define TR2_OPTIONAL_GCC_4_8_1_AND_HIGHER___ # endif # endif foo """ output = r"""#line 23 foo """ class test15(unittest.TestCase, runner): input = r"""#define f(type) type type##_base f(g) """ output = r""" g g_base """ class test16(unittest.TestCase, runner): # #if ((1?2:3) == 2) is known to fail input = r"""#if (((1)?2:3) == 2) hi #endif """ output = r""" hi """ class test17(unittest.TestCase, runner): input = r"""#if L'\0' == 0 hi #endif """ output = r""" hi """ class test18(unittest.TestCase, runner): input = r""" /* multiline comment */ void shouldBeOnLineSeven(); """ output = r""" void shouldBeOnLineSeven(); """ class test19(unittest.TestCase, runner): input = r""" /* a comment that spans eight lines */ void shouldBeOnLineEleven();""" output = r"""#line 11 void shouldBeOnLineEleven(); """ class test20(unittest.TestCase, runner): input = r""" #define PASTE(x, y) x ## y #if PASTE(1, 2) == 12 works #else fails #endif """ output = r""" works """ class test21(unittest.TestCase, runner): input = r""" #define PASTE(x, y, z) x ## y ## z #if PASTE(1, 2, L) == 12 works #else fails #endif """ output = r""" works """ class test22(unittest.TestCase, runner): input = r""" #define OUTCOME_TRY_GLUE2(x, y) x##y #define OUTCOME_TRY_GLUE(x, y) OUTCOME_TRY_GLUE2(x, y) #define OUTCOME_TRY_UNIQUE_NAME OUTCOME_TRY_GLUE(_outcome_try_unique_name_temporary, __LINE__) OUTCOME_TRY_UNIQUE_NAME OUTCOME_TRY_UNIQUE_NAME OUTCOME_TRY_UNIQUE_NAME """ output = r""" _outcome_try_unique_name_temporary6 _outcome_try_unique_name_temporary7 _outcome_try_unique_name_temporary8 """ class test23(unittest.TestCase, runner): input = r""" #define FUNC1(rettype) rettype #define FUNC2 void FUNC1(void)foo() { } FUNC2|foo() { } FUNC1(void)/foo() { } FUNC1(void)FUNC2.FUNC1(void)foo() { }""" output = r""" void foo() { } void|foo() { } void/foo() { } void void.void foo() { } """ class test24(unittest.TestCase, runner): input = r""" const char *foo = "Hello"\ "Niall"\ ;""" output = r""" const char *foo = "Hello""Niall"; """ class test25(unittest.TestCase, runner): input = r""" const char *foo = "Niall \ says\ hello"\;""" output = r""" const char *foo = "Niall sayshello"\; """ class test26(unittest.TestCase, runner): input = r""" #if BOO(FOO) fail #else success #endif """ output = r""" success """ class test27(unittest.TestCase, runner): input = r""" #define BOOST_WORKAROUND(symbol, test) \ ((symbol ## _WORKAROUND_GUARD + 0 == 0) && \ (symbol != 0) && (1 % (( (symbol test) ) + 1))) #define BOOST_MSVC 1916 #define BOOST_MSVC_WORKAROUND_GUARD 0 BOOST_WORKAROUND(BOOST_MSVC,==1916) """ output = r"""#line 8 ((0 + 0 == 0) && (1916 != 0) && (1 % (( (1916 ==1916) ) + 1))) """ class test28(unittest.TestCase, runner): input = r""" #define TEST \ 1 + 2 \ 3 + 4 TEST """ output = r""" 1 + 2 3 + 4 """ class test29(unittest.TestCase, runner): input = r""" #define fCAST4_8s(A) ((int64_t)((int32_t)(A))) #define fBIDIR_SHIFTR(SRC, SHAMT, REGSTYPE) (((SHAMT) < 0) ? ((fCAST##REGSTYPE(SRC) << ((-(SHAMT)) - 1)) << 1) : (fCAST##REGSTYPE(SRC) >> (SHAMT))) #define fBIDIR_ASHIFTR(SRC, SHAMT, REGSTYPE) fBIDIR_SHIFTR(SRC, SHAMT, REGSTYPE##s) #define fSXTN(N, M, VAL) (((N) != 0) ? sextract64((VAL), 0, (N)) : 0LL) #define fHIDE(A) A #define fECHO(A) (A) #define DEF_SHORTCODE(TAG,SHORTCODE) insn(TAG, SHORTCODE) DEF_SHORTCODE(S2_asr_r_r_acc, { fHIDE(size4s_t) shamt=fSXTN(7,32,RtV); RxV = fECHO(RxV + fBIDIR_ASHIFTR(RsV,shamt,4_8)); }) """ output = r"""#line 10 insn(S2_asr_r_r_acc, { size4s_t shamt=(((7) != 0) ? sextract64((RtV), 0, (7)) : 0LL); RxV = (RxV + (((shamt) < 0) ? ((((int64_t)((int32_t)(RsV))) << ((-(shamt)) - 1)) << 1) : (((int64_t)((int32_t)(RsV))) >> (shamt)))); }) """ class test30(unittest.TestCase, runner): input = r""" #define FOO(x) x #define BAR FOO(BAR) BAR """ output = r""" BAR """ class test31(unittest.TestCase, runner): input = r""" #define PCRE2_SIZE size_t #define PCRE2_COMPILE_FUNCTIONS \ pcre2_code *pcre2_compile(PCRE2_SPTR, PCRE2_SIZE, uint32_t, int *, PCRE2_SIZE *, \ pcre2_compile_context *); #define PCRE2_JOIN(a,b) a ## b #define PCRE2_GLUE(a,b) PCRE2_JOIN(a,b) #define PCRE2_SUFFIX(a) PCRE2_GLUE(a,PCRE2_LOCAL_WIDTH) #define PCRE2_SPTR PCRE2_SUFFIX(PCRE2_SPTR) #define pcre2_code PCRE2_SUFFIX(pcre2_code_) #define pcre2_compile_context PCRE2_SUFFIX(pcre2_compile_context_) #define pcre2_compile PCRE2_SUFFIX(pcre2_compile_) #define PCRE2_TYPES_STRUCTURES_AND_FUNCTIONS \ PCRE2_COMPILE_FUNCTIONS #define PCRE2_LOCAL_WIDTH 8 PCRE2_TYPES_STRUCTURES_AND_FUNCTIONS #undef PCRE2_LOCAL_WIDTH #define PCRE2_LOCAL_WIDTH 16 PCRE2_TYPES_STRUCTURES_AND_FUNCTIONS #undef PCRE2_LOCAL_WIDTH #define PCRE2_LOCAL_WIDTH 32 PCRE2_TYPES_STRUCTURES_AND_FUNCTIONS #undef PCRE2_LOCAL_WIDTH """ output = r"""#line 21 pcre2_code_8 *pcre2_compile_8(PCRE2_SPTR8, size_t, uint32_t, int *, size_t *, pcre2_compile_context_8 *); pcre2_code_16 *pcre2_compile_16(PCRE2_SPTR16, size_t, uint32_t, int *, size_t *, pcre2_compile_context_16 *); pcre2_code_32 *pcre2_compile_32(PCRE2_SPTR32, size_t, uint32_t, int *, size_t *, pcre2_compile_context_32 *); """ if __name__ == '__main__': unittest.main() ================================================ FILE: tests/doctests.py ================================================ import unittest class pcpp_doctests(unittest.TestCase): def runTest(self): import doctest, pcpp.preprocessor, pcpp.evaluator failurecount, testcount = doctest.testmod(pcpp.evaluator) self.assertGreater(testcount, 0) self.assertEqual(failurecount, 0) failurecount, testcount = doctest.testmod(pcpp.preprocessor) #self.assertGreater(testcount, 0) self.assertEqual(failurecount, 0) ================================================ FILE: tests/embedded.py ================================================ import unittest, sys from io import StringIO class embedded1(unittest.TestCase): def runTest(self): from pcpp import Preprocessor output = r''' a ''' p = Preprocessor() p.define('BAR FOO') p.parse(r'''#define FOO 1 #if FOO == BAR a #endif ''') oh = StringIO() p.write(oh) if oh.getvalue() != output: print("Should be:\n" + output, file = sys.stderr) print("\n\nWas:\n" + oh.getvalue(), file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(oh.getvalue(), output) ================================================ FILE: tests/eval.py ================================================ import unittest, time from io import StringIO clock = time.process_time class runner(object): def runTest(self): from pcpp import Preprocessor import os, sys start = clock() #p = Preprocessor() #lines = p.group_lines(self.input, '') #for x in lines: # print(x) p = Preprocessor() p.parse(self.input) oh = StringIO() p.write(oh) end = clock() print("Preprocessed test in", end-start, "seconds") if oh.getvalue() != self.output: print("Should be:\n" + self.output, file = sys.stderr) print("\n\nWas:\n" + oh.getvalue(), file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(oh.getvalue(), self.output) class eval1(unittest.TestCase, runner): input = r"""#if -1 >= 0U correct #endif""" output = r""" correct """ class eval2(unittest.TestCase, runner): input = r"""#if 1<<2 == 4 correct #endif""" output = r""" correct """ class eval3(unittest.TestCase, runner): input = r"""#if (-!+!9) == -1 correct #endif""" output = r""" correct """ class eval4(unittest.TestCase, runner): input = r"""#if (2 || 3) == 1 correct #endif""" output = r""" correct """ ================================================ FILE: tests/issue0017/inc.h ================================================ inc1 #ifndef __inc_h__ inc2 #define __inc_h__ inc3 #endif //__inc_h__ inc4 ================================================ FILE: tests/issue0017/issue0017.c ================================================ #include "inc.h" test1 #include "inc.h" test2 ================================================ FILE: tests/issue0017-ref.i ================================================ #line 1 "tests/issue0017/inc.h" inc1 inc2 inc3 inc4 #line 2 "tests/issue0017/issue0017.c" test1 #line 1 "tests/issue0017/inc.h" inc1 inc4 #line 4 "tests/issue0017/issue0017.c" test2 ================================================ FILE: tests/issue0017.py ================================================ import unittest, os class issue0017(unittest.TestCase): def runTest(self): from pcpp import Preprocessor import os, sys p = Preprocessor() path = 'tests/issue0017/issue0017.c' with open(path, 'rt') as ih: p.parse(ih.read(), path) with open('tests/issue0017.i', 'w') as oh: p.write(oh) with open('tests/issue0017.i', 'r') as ih: was = ih.read() os.remove('tests/issue0017.i') with open('tests/issue0017-ref.i', 'r') as ih: shouldbe = ih.read() if was != shouldbe: print("Should be:\n" + shouldbe, file = sys.stderr) print("\n\nWas:\n" + was, file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(was, shouldbe) ================================================ FILE: tests/issue0025/inc.h ================================================ OUTCOME_TRY_UNIQUE_NAME __LINE__ OUTCOME_TRY_UNIQUE_NAME __LINE__ ================================================ FILE: tests/issue0025/main.c ================================================ #define OUTCOME_TRY_GLUE2(x, y) x##y #define OUTCOME_TRY_GLUE(x, y) OUTCOME_TRY_GLUE2(x, y) #define OUTCOME_TRY_UNIQUE_NAME OUTCOME_TRY_GLUE(_outcome_try_unique_name_temporary, __LINE__) OUTCOME_TRY_UNIQUE_NAME __LINE__ OUTCOME_TRY_UNIQUE_NAME __LINE__ OUTCOME_TRY_UNIQUE_NAME __LINE__ #include "inc.h" OUTCOME_TRY_UNIQUE_NAME __LINE__ #include "inc.h" OUTCOME_TRY_UNIQUE_NAME __LINE__ ================================================ FILE: tests/issue0025-ref.i ================================================ #line 5 "tests/issue0025/main.c" _outcome_try_unique_name_temporary5 5 _outcome_try_unique_name_temporary6 6 _outcome_try_unique_name_temporary7 7 #line 1 "tests/issue0025/inc.h" _outcome_try_unique_name_temporary1 1 _outcome_try_unique_name_temporary3 3 #line 10 "tests/issue0025/main.c" _outcome_try_unique_name_temporary10 10 #line 1 "tests/issue0025/inc.h" _outcome_try_unique_name_temporary1 1 _outcome_try_unique_name_temporary3 3 #line 12 "tests/issue0025/main.c" _outcome_try_unique_name_temporary12 12 ================================================ FILE: tests/issue0025.py ================================================ import unittest, os class issue0025(unittest.TestCase): def runTest(self): from pcpp import Preprocessor import os, sys p = Preprocessor() path = 'tests/issue0025/main.c' with open(path, 'rt') as ih: p.parse(ih.read(), path) with open('tests/issue0025.i', 'w') as oh: p.write(oh) with open('tests/issue0025.i', 'r') as ih: was = ih.read() os.remove('tests/issue0025.i') with open('tests/issue0025-ref.i', 'r') as ih: shouldbe = ih.read() if was != shouldbe: print("Should be:\n" + shouldbe, file = sys.stderr) print("\n\nWas:\n" + was, file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(was, shouldbe) ================================================ FILE: tests/issue0027.py ================================================ import unittest, time from io import StringIO clock = time.process_time class runner(object): def runTest(self): from pcpp import Preprocessor, OutputDirective, Action import os, sys class PassThruPreprocessor(Preprocessor): def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks): if len(precedingtoks) == 1: # Execute return super(PassThruPreprocessor, self).on_directive_handle(directive,toks,ifpassthru,precedingtoks) raise OutputDirective(Action.IgnoreAndPassThrough) start = clock() p = PassThruPreprocessor() p.parse(self.input) oh = StringIO() p.write(oh) end = clock() print("Preprocessed test in", end-start, "seconds") if oh.getvalue() != self.output: print("Should be:\n" + self.output + "EOF\n", file = sys.stderr) print("\nWas:\n" + oh.getvalue()+"EOF\n", file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(oh.getvalue(), self.output) class space_after_hash(unittest.TestCase, runner): input = r"""#if 5 1 #endif # if 5 2 # endif #warning Hi # warning Hi2""" output = r""" 1 # if 5 2 # endif # warning Hi2 """ ================================================ FILE: tests/issue0030/source1.c ================================================ #undef FOO #define FOO 1 ================================================ FILE: tests/issue0030/source2.c ================================================ #undef FOO #define FOO 2 ================================================ FILE: tests/issue0030/source3.c ================================================ FOO ================================================ FILE: tests/issue0030.py ================================================ import unittest, sys, os shouldbe = r'''#line 1 "tests/issue0030/source3.c" 2 ''' class runner(object): def runTest(self): from pcpp import CmdPreprocessor p = CmdPreprocessor(['pcpp', '-o', 'tests/issue0030.c', 'tests/issue0030/source1.c', 'tests/issue0030/source2.c', 'tests/issue0030/source3.c']) with open('tests/issue0030.c', 'rt') as ih: output = ih.read() os.remove('tests/issue0030.c') if output != shouldbe: print("Should be:\n" + shouldbe + "EOF\n", file = sys.stderr) print("\nWas:\n" + output + "EOF\n", file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(output, shouldbe) class multiple_input_files(unittest.TestCase, runner): pass ================================================ FILE: tests/issue0032.py ================================================ import unittest, sys shouldbe = r'''#line 1 "tests/issue0030/source3.c" 2 ''' class runner(object): def runTest(self): from pcpp import CmdPreprocessor p = CmdPreprocessor(['pcpp', '--time', 'tests/issue0030/source1.c']) self.assertEqual(p.return_code, 0) class no_output_file(unittest.TestCase, runner): pass ================================================ FILE: tests/issue0037/inc.h ================================================ /** this spans two lines */ virtual std::string baseOnly(); ================================================ FILE: tests/issue0037.py ================================================ import unittest, sys, os shouldbe = r'''#line 1 "tests/issue0037/inc.h" /** this spans two lines */ virtual std::string baseOnly(); ''' class runner(object): def runTest(self): from pcpp import CmdPreprocessor p = CmdPreprocessor(['pcpp', '--time', '--passthru-comments', '-o', 'tests/issue0037.i', 'tests/issue0037/inc.h']) with open('tests/issue0037.i', 'rt') as ih: output = ih.read() os.remove('tests/issue0037.i') if output != shouldbe: print("Should be:\n" + shouldbe + "EOF\n", file = sys.stderr) print("\nWas:\n" + output + "EOF\n", file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(output, shouldbe) class multiline_comments(unittest.TestCase, runner): pass ================================================ FILE: tests/issue0044.h ================================================ a1 #if 0 sample text #endif b1 ================================================ FILE: tests/issue0044.py ================================================ import unittest import sys, os shouldbe = r'''a1 b1 ''' class runner(object): def runTest(self): from pcpp import CmdPreprocessor # failure: p = CmdPreprocessor(['pcpp', '--line-directive', '#line', # p = CmdPreprocessor(['pcpp', '--line-directive', 'nothing', # p = CmdPreprocessor(['pcpp', '--line-directive', 'None', p = CmdPreprocessor(['pcpp', '--line-directive', '', '-o', 'tests/issue0044.i', 'tests/issue0044.h']) with open('tests/issue0044.i', 'rt') as ih: output = ih.read() os.remove('tests/issue0044.i') if output != shouldbe: print("Should be:\n" + shouldbe + "EOF\n", file=sys.stderr) print("\nWas:\n" + output + "EOF\n", file=sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(output, shouldbe) class empty_line_directive(unittest.TestCase, runner): pass ================================================ FILE: tests/issue0051.c ================================================ #include "issue0051.h" #ifdef FOO TRUE #else FALSE #endif ================================================ FILE: tests/issue0051.h ================================================ #define FOO 1 void my_func1(); void my_func2(); void my_func3(); ================================================ FILE: tests/issue0051.py ================================================ import unittest import sys, os class runner(object): def runTest(self): from pcpp import CmdPreprocessor p = CmdPreprocessor(['pcpp'] + self.options + [ '-o', 'tests/issue0051.i', 'tests/issue0051.c']) with open('tests/issue0051.i', 'rt') as ih: output = ih.read() os.remove('tests/issue0051.i') if output != self.shouldbe: print("Should be:\n" + self.shouldbe + "EOF\n", file=sys.stderr) print("\nWas:\n" + output + "EOF\n", file=sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(output, self.shouldbe) class normal_inclusion(unittest.TestCase, runner): options = [] shouldbe = r'''#line 3 "tests/issue0051.h" void my_func1(); void my_func2(); void my_func3(); #line 4 "tests/issue0051.c" TRUE ''' class exclude_inclusion(unittest.TestCase, runner): options = ['--passthru-includes', '"issue0051.h"'] shouldbe = r'''#line 1 "tests/issue0051.c" #include "issue0051.h" TRUE ''' ================================================ FILE: tests/issue0057.h ================================================ headertoken ================================================ FILE: tests/issue0057.py ================================================ import unittest, time from io import StringIO clock = time.process_time class runner(object): def runTest(self): from pcpp import Preprocessor, OutputDirective, Action import os, sys, re class PassThruPreprocessor(Preprocessor): def __init__(self): super(PassThruPreprocessor, self).__init__() self.passthru_includes = re.compile('.*/issue0057.*') def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath): raise OutputDirective(Action.IgnoreAndPassThrough) start = clock() p = PassThruPreprocessor() p.parse(self.input) oh = StringIO() p.write(oh) end = clock() print("Preprocessed test in", end-start, "seconds") if oh.getvalue() != self.output: print("Should be:\n" + self.output + "EOF\n", file = sys.stderr) print("\nWas:\n" + oh.getvalue()+"EOF\n", file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(oh.getvalue(), self.output) class newline_after_passthru_include1(unittest.TestCase, runner): input = r"""#include "tests/issue0057.h" filetoken1 #include "tests/issue0057.h" filetoken2 """ output = r"""#include "tests/issue0057.h" filetoken1 #include "tests/issue0057.h" filetoken2 """ class newline_after_passthru_include2(unittest.TestCase, runner): input = r"""#include "tests/issue0057x.h" filetoken1 #include "tests/issue0057x.h" filetoken2 """ output = r"""#include "tests/issue0057x.h" filetoken1 #include "tests/issue0057x.h" filetoken2 """ class newline_after_passthru_include3(unittest.TestCase, runner): input = r"""#include "unfoundfile" filetoken1 #include "unfoundfile" filetoken2 """ output = r"""#include "unfoundfile" filetoken1 #include "unfoundfile" filetoken2 """ ================================================ FILE: tests/issue0059.py ================================================ import unittest, time from io import StringIO clock = time.process_time class runner(object): def runTest(self): from pcpp import Preprocessor, OutputDirective, Action import os, sys, re class PassThruPreprocessor(Preprocessor): def __init__(self): super(PassThruPreprocessor, self).__init__() self.passthru_includes = re.compile('.*tests/issue0059[a-z]\\.h"') def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath): raise OutputDirective(Action.IgnoreAndPassThrough) start = clock() p = PassThruPreprocessor() p.parse(self.input) oh = StringIO() p.write(oh) end = clock() print("Preprocessed test in", end-start, "seconds") if oh.getvalue() != self.output: print("Should be:\n" + self.output + "EOF\n", file = sys.stderr) print("\nWas:\n" + oh.getvalue()+"EOF\n", file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(oh.getvalue(), self.output) class multiple_inclusion(unittest.TestCase, runner): input = r"""#include "tests/issue0059a.h" FOO #include "tests/issue0059b.h" FOO #include "tests/issue0059a.h" FOO #include "tests/issue0059b.h" FOO #include "tests/issue0059a.h" FOO #include "tests/issue0059b.h" FOO """ output = r"""#include "tests/issue0059a.h" 0 #include "tests/issue0059b.h" FOO #include "tests/issue0059a.h" FOO #include "tests/issue0059b.h" FOO #include "tests/issue0059a.h" FOO #include "tests/issue0059b.h" FOO """ ================================================ FILE: tests/issue0059a.h ================================================ #pragma once #define FOO __COUNTER__ ================================================ FILE: tests/issue0059b.h ================================================ #undef FOO ================================================ FILE: tests/issue0063.c ================================================ #define x\ \ #include "issue0063.h" #undef x #define x\ \ #include "issue0063.h" ================================================ FILE: tests/issue0063.h ================================================ int f(); ================================================ FILE: tests/issue0063.py ================================================ import unittest import sys, os class runner(object): def runTest(self): from pcpp import CmdPreprocessor p = CmdPreprocessor(['pcpp'] + self.options + [ '-o', 'tests/issue0063.i', 'tests/issue0063.c']) with open('tests/issue0063.i', 'rt') as ih: output = ih.read() os.remove('tests/issue0063.i') if output != self.shouldbe: print("Should be:\n" + self.shouldbe + "EOF\n", file=sys.stderr) print("\nWas:\n" + output + "EOF\n", file=sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(output, self.shouldbe) class include_after_continued_macro1(unittest.TestCase, runner): options = [] shouldbe = r'''#line 1 "tests/issue0063.h" int f(); int f(); ''' class include_after_continued_macro2(unittest.TestCase, runner): options = [ '--passthru-defines' ] shouldbe = r'''#line 1 "tests/issue0063.c" #define x #line 1 "tests/issue0063.h" int f(); #line 6 "tests/issue0063.c" #undef x #define x #line 1 "tests/issue0063.h" int f(); ''' class include_after_continued_macro3(unittest.TestCase, runner): options = [ '--line-directive=', '--passthru-defines' ] shouldbe = r'''#define x int f(); #undef x #define x int f(); ''' ================================================ FILE: tests/issue0079.py ================================================ import unittest, time from io import StringIO class runner(object): def runTest(self): from pcpp import Preprocessor, OutputDirective, Action import os, sys, re p = Preprocessor() p.parse(self.input) oh = StringIO() p.write(oh) if oh.getvalue() != self.output: print("Should be:\n" + self.output + "EOF\n", file = sys.stderr) print("\nWas:\n" + oh.getvalue()+"EOF\n", file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(oh.getvalue(), self.output) class pp_number_pasting(unittest.TestCase, runner): input = r"""#define CAT_(A,B)A##B #define CAT(A,B)CAT_(A,B) #define Ox 0x #if CAT(Ox,1) PP_NUMBER pasted ok #else PP_NUMBER paste fail #endif """ output = r""" PP_NUMBER pasted ok """ ================================================ FILE: tests/issue0098/dir1/header.h ================================================ header1 #include_next "header.h" ================================================ FILE: tests/issue0098/dir2/header.h ================================================ header2 #include_next "header.h" ================================================ FILE: tests/issue0098/dir3/header.h ================================================ header3 #include_next "header.h" ================================================ FILE: tests/issue0098/dir4/header.h ================================================ header4 ================================================ FILE: tests/issue0098.py ================================================ import unittest, sys from io import StringIO class runner(object): def runTest(self): from pcpp import Preprocessor p = Preprocessor() p.include_next_enabled = True # Add the test directories to the search path p.add_path('tests/issue0098/dir1') p.add_path('tests/issue0098/dir2') p.add_path('tests/issue0098/dir3') p.add_path('tests/issue0098/dir4') p.parse(self.input) output = StringIO() p.write(output) if output.getvalue() != self.shouldbe: print("Should be:\n" + self.shouldbe + "EOF\n", file=sys.stderr) print("\nWas:\n" + output.getvalue() + "EOF\n", file=sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(output.getvalue(), self.shouldbe) class include_next_works(unittest.TestCase, runner): input = r'''#include_next "header.h" ''' shouldbe = r'''#line 1 "tests/issue0098/dir1/header.h" header1 #line 1 "tests/issue0098/dir2/header.h" header2 #line 1 "tests/issue0098/dir3/header.h" header3 #line 1 "tests/issue0098/dir4/header.h" header4 ''' class has_include_works(unittest.TestCase, runner): input = r'''#ifdef __has_include ifdef #endif #ifndef __has_include ifndef #endif #if defined(__has_include) defined #endif #if __has_include("header.h") header #endif ''' shouldbe = r''' ifdef defined header ''' if __name__ == '__main__': unittest.main() ================================================ FILE: tests/issue0103.py ================================================ import unittest, sys from io import StringIO class runner(object): def runTest(self): from pcpp import Preprocessor p = Preprocessor() p.parse(self.input) output = StringIO() p.write(output) if output.getvalue() != self.shouldbe: print("Should be:\n" + self.shouldbe + "EOF\n", file=sys.stderr) print("\nWas:\n" + output.getvalue() + "EOF\n", file=sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(output.getvalue(), self.shouldbe) class multiline_char_literals1(unittest.TestCase, runner): input = r'''#if 'N\ \ \ \ \ \ \ \ ' == 78 FOO #endif ''' shouldbe = r'''#line 10 FOO ''' class multiline_char_literals2(unittest.TestCase, runner): input = r'''#if '\ \ \ \ \ \ \ \ N' == 78 FOO #endif ''' shouldbe = r'''#line 10 FOO ''' if __name__ == '__main__': unittest.main() ================================================ FILE: tests/n_std-clang.i ================================================ # 1 "tests/test-c/n_std.c" # 1 "" 1 # 1 "" 3 # 312 "" 3 # 1 "" 1 # 1 "" 2 # 1 "tests/test-c/n_std.c" 2 # 18 "tests/test-c/n_std.c" # 1 "tests/test-c/defs.h" 1 # 19 "tests/test-c/n_std.c" 2 # 30 "tests/test-c/n_std.c" void n_1( void); void n_2( void); void n_3( void); void n_4( void); void n_5( void); void n_6( void); void n_7( void); void n_9( void); void n_10( void); void n_11( void); void n_12( void); void n_13( void); void n_13_5( void); void n_13_7( void); void n_13_8( void); void n_13_13( void); void n_15( void); void n_18( void); void n_19( void); void n_20( void); void n_21( void); void n_22( void); void n_23( void); void n_24( void); void n_25( void); void n_26( void); void n_27( void); void n_28( void); void n_29( void); void n_30( void); void n_32( void); void n_37( void); int main( void) { n_2(); n_3(); n_5(); n_6(); n_7(); n_9(); n_10(); n_11(); n_12(); n_13(); n_13_5(); n_13_7(); n_13_8(); n_13_13(); n_15(); n_18(); n_19(); n_20(); n_21(); n_22(); n_23(); n_24(); n_25(); n_26(); n_27(); n_28(); n_29(); n_30(); n_32(); n_37(); puts( ""); return 0; } char quasi_trigraph[] = { '?', '?', ' ', '?', '?', '?', ' ' , '?', '?', '%', ' ', '?', '?', '^', ' ', '?', '#', '\0' }; void n_2( void) { int ab = 1, cd = 2, ef = 3, abcde = 5; assert( ab + cd + ef == 6); # 122 "tests/test-c/n_std.c" assert (ab + cd + ef == 6); assert (strcmp( "abcde", "abcde") == 0); assert( abcde == 5); } void n_3( void) { int abcd = 4; assert( strcmp( "abc de", "abc de") == 0); # 156 "tests/test-c/n_std.c" assert( abcd == 4); } void n_5( void) { int abcde = 5; assert( abcde == 5); } # 178 "tests/test-c/n_std.c" void n_6( void) { int abc = 3; assert( isalpha( 'a')); # 1 "tests/test-c/header.h" 1 # 188 "tests/test-c/n_std.c" 2 assert( abc == 3); # 1 "tests/test-c/header.h" 1 # 193 "tests/test-c/n_std.c" 2 assert( abc == 3); } void n_7( void) { # 1234 "cpp" assert( 1234 == 1234); assert( strcmp( "cpp", "cpp") == 0); # 2345 "cpp" assert( 2345 == 2345); assert( strcmp( "cpp", "cpp") == 0); # 1234 "n_7.c" assert( 1234 == 1234); assert( strcmp( "n_7.c", "n_7.c") == 0); } # 218 "n_std.c" void n_9( void) { #pragma who knows ? } void n_10( void) { assert( 1); # 251 "n_std.c" } void n_11( void) { int abc = 1, a = 0; assert( abc); assert( abc); # 279 "n_std.c" } void n_12( void) { fputs( "Bad evaluation of long.\n", stderr); fputs( "Bad evaluation of long.\n", stderr); # 325 "n_std.c" } void n_13( void) # 343 "n_std.c" { # 363 "n_std.c" } void n_13_5( void) { # 387 "n_std.c" } void n_13_7( void) { # 406 "n_std.c" } void n_13_8( void) { # 439 "n_std.c" } void n_13_13( void) { # 466 "n_std.c" } void n_15( void) { assert( 1); # 483 "n_std.c" assert( 1); } void n_18( void) { int c = 3; assert( (1-1) == 0); assert( ( c ) == 3); assert( strcmp( "n1:n2", "n1:n2") == 0); } void n_19( void) { int c = 1; # 521 "n_std.c" assert( ( c ) == 1); } void n_20( void) { double fl; assert( sizeof fl == sizeof (double)); } void n_21( void) { int a = 1, x = 2, y = -3; assert( - - -a == -1); assert( x- -y == -1); } void n_22( void) { assert( strcmp( "12E+EXP", "12E+EXP") == 0); assert( strcmp( ".2e-EXP", ".2e-EXP") == 0); assert( strcmp( "12+1", "12+1") == 0); } void n_23( void) { int xy = 1; assert( xy == 1); assert( .12e+2 == 12.0); } void n_24( void) { assert( strcmp( "a+b", "a+b") == 0); assert( strcmp( "ab + cd", "ab + cd") == 0); assert( strcmp( "'\"' + \"' \\\"\"", "'\"' + \"' \\\"\"") == 0); assert( strcmp( "\"abc\"", "\"abc\"") == 0); assert( strcmp( "x-y", "x-y") == 0); } void n_25( void) { int a = 1, b = 2, abc = 3, MACRO_0MACRO_1 = 2; assert( (a,b - 1) == 1); assert( ( - a) == -1); assert( abc == 3); assert( MACRO_0MACRO_1 == 2); assert( strcmp( "ZERO_TOKEN", "ZERO_TOKEN") == 0); } # 648 "n_std.c" int f( int a) { return a; } int g( int a) { return a * 2; } void n_26( void) { int x = 1; int AB = 1; int Z[1]; Z[0] = 1; assert( Z[0] == 1); assert( AB == 1); assert( x + f(x) == 2); assert( x + x + g( x) == 4); assert( Z[0] + f(Z[0]) == 2); } void n_27( void) { int a = 1, b = 2, c, m = 1, n = 2; # 711 "n_std.c" assert( 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 == 36); assert( (1) + (1 + 2) + 1 + 2 + 1 + 2 + 3 + 1 + 2 + 3 + 4 == 23); assert( 1 == 1); assert( ((a) - (b)) == -1); c = (a - b); assert( c == -1); assert( n == 2); } void n_28( void) { char * date = "Jan 13 2020"; assert( strcmp( "n_std.c", "n_std.c") == 0); assert( 751 == 779); assert( strlen( "Jan 13 2020") == 11); assert( date[ 4] != '0'); assert( strlen( "10:47:38") == 8); assert( 1); assert( 199901L >= 199409L); # 1 "tests/test-c/line.h" 1 { char * file = "tests/test-c/line.h"; file += strlen( file) - 6; assert( 6 == 6 && strcmp( file, "line.h") == 0); } # 768 "n_std.c" 2 } void n_29( void) { int DEFINED = 1; assert( DEFINED == 1); } void n_30( void) { int a = 1, b = 2, c = 3; assert ( a + b + c == 6 ); } void n_32( void) { # 822 "n_std.c" } void n_37( void) { int ABCDEFGHIJKLMNOPQRSTUVWXYZabcde = 31; int ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ = 30; int nest = 0; assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcde == 31); assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ == 30); nest = 0; # 865 "n_std.c" nest = 8; # 874 "n_std.c" assert( nest == 8); nest = 0; # 1 "tests/test-c/nest1.h" 1 nest = 1; # 1 "tests/test-c/nest2.h" 1 nest = 2; # 1 "tests/test-c/nest3.h" 1 nest = 3; # 1 "tests/test-c/nest4.h" 1 nest = 4; # 1 "tests/test-c/nest5.h" 1 nest = 5; # 1 "tests/test-c/nest6.h" 1 nest = 6; # 1 "tests/test-c/nest7.h" 1 nest = 7; # 1 "tests/test-c/nest8.h" 1 nest = 8; # 6 "tests/test-c/nest7.h" 2 # 6 "tests/test-c/nest6.h" 2 # 6 "tests/test-c/nest5.h" 2 # 6 "tests/test-c/nest4.h" 2 # 6 "tests/test-c/nest3.h" 2 # 6 "tests/test-c/nest2.h" 2 # 6 "tests/test-c/nest1.h" 2 # 879 "n_std.c" 2 assert( nest == 8); nest = 32; assert( nest == 32); { char * extremely_long_string = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567" ; assert( strlen( extremely_long_string) == 507); } { int a123456789012345678901234567890 = 123450; int b123456789012345678901234567890 = 123451; int c123456789012345678901234567890 = 123452; int d123456789012345678901234567890 = 123453; int e123456789012345678901234567890 = 123454; int f123456789012345678901234567890 = 123455; int A123456789012345678901234567890 = 123456; int B123456789012345678901234567890 = 123457; int C123456789012345678901234567890 = 123458; int D1234567890123456789012 = 123459; # 917 "n_std.c" assert( a123456789012345678901234567890 == 123450 && D1234567890123456789012 == 123459); } # 1 "tests/test-c/m1024.h" 1 # 927 "n_std.c" 2 assert( 1); } ================================================ FILE: tests/n_std-gcc.i ================================================ # 1 "tests/test-c/n_std.c" # 1 "" # 1 "" # 1 "/usr/include/stdc-predef.h" 1 3 4 # 1 "" 2 # 1 "tests/test-c/n_std.c" # 18 "tests/test-c/n_std.c" # 1 "tests/test-c/defs.h" 1 # 19 "tests/test-c/n_std.c" 2 # 30 "tests/test-c/n_std.c" void n_1( void); void n_2( void); void n_3( void); void n_4( void); void n_5( void); void n_6( void); void n_7( void); void n_9( void); void n_10( void); void n_11( void); void n_12( void); void n_13( void); void n_13_5( void); void n_13_7( void); void n_13_8( void); void n_13_13( void); void n_15( void); void n_18( void); void n_19( void); void n_20( void); void n_21( void); void n_22( void); void n_23( void); void n_24( void); void n_25( void); void n_26( void); void n_27( void); void n_28( void); void n_29( void); void n_30( void); void n_32( void); void n_37( void); int main( void) { n_2(); n_3(); n_5(); n_6(); n_7(); n_9(); n_10(); n_11(); n_12(); n_13(); n_13_5(); n_13_7(); n_13_8(); n_13_13(); n_15(); n_18(); n_19(); n_20(); n_21(); n_22(); n_23(); n_24(); n_25(); n_26(); n_27(); n_28(); n_29(); n_30(); n_32(); n_37(); puts( ""); return 0; } char quasi_trigraph[] = { '?', '?', ' ', '?', '?', '?', ' ' , '?', '?', '%', ' ', '?', '?', '^', ' ', '?', '#', '\0' }; void n_2( void) { int ab = 1, cd = 2, ef = 3, abcde = 5; assert( ab + cd + ef == 6); # 122 "tests/test-c/n_std.c" assert (ab + cd + ef == 6); assert (strcmp( "abcde", "abcde") == 0); assert( abcde == 5); } void n_3( void) { int abcd = 4; assert( strcmp( "abc de", "abc de") == 0); # 156 "tests/test-c/n_std.c" assert( abcd == 4); } void n_5( void) { int abcde = 5; assert( abcde == 5); } # 178 "tests/test-c/n_std.c" void n_6( void) { int abc = 3; assert( isalpha( 'a')); # 1 "tests/test-c/header.h" 1 # 188 "tests/test-c/n_std.c" 2 assert( abc == 3); # 1 "tests/test-c/header.h" 1 # 193 "tests/test-c/n_std.c" 2 assert( abc == 3); } void n_7( void) { # 1234 "cpp" assert( 1234 == 1234); assert( strcmp( "cpp", "cpp") == 0); # 2345 "cpp" assert( 2345 == 2345); assert( strcmp( "cpp", "cpp") == 0); # 1234 "n_7.c" assert( 1234 == 1234); assert( strcmp( "n_7.c", "n_7.c") == 0); } # 218 "n_std.c" void n_9( void) { #pragma who knows ? } void n_10( void) { assert( 1); # 251 "n_std.c" } void n_11( void) { int abc = 1, a = 0; assert( abc); assert( abc); # 279 "n_std.c" } void n_12( void) { fputs( "Bad evaluation of long.\n", stderr); fputs( "Bad evaluation of long.\n", stderr); # 325 "n_std.c" } void n_13( void) # 343 "n_std.c" { # 363 "n_std.c" } void n_13_5( void) { # 387 "n_std.c" } void n_13_7( void) { # 406 "n_std.c" } void n_13_8( void) { # 439 "n_std.c" } void n_13_13( void) { # 466 "n_std.c" } void n_15( void) { assert( 1); # 483 "n_std.c" assert( 1); } void n_18( void) { int c = 3; assert( (1-1) == 0); assert( ( c ) == 3); assert( strcmp( "n1:n2", "n1:n2") == 0); } void n_19( void) { int c = 1; # 521 "n_std.c" assert( ( c ) == 1); } void n_20( void) { double fl; assert( sizeof fl == sizeof (double)); } void n_21( void) { int a = 1, x = 2, y = -3; assert( - - -a == -1); assert( x- -y == -1); } void n_22( void) { assert( strcmp( "12E+EXP", "12E+EXP") == 0); assert( strcmp( ".2e-EXP", ".2e-EXP") == 0); assert( strcmp( "12+1", "12+1") == 0); } void n_23( void) { int xy = 1; assert( xy == 1); assert( .12e+2 == 12.0); } void n_24( void) { assert( strcmp( "a+b", "a+b") == 0); assert( strcmp( "ab + cd" , "ab + cd") == 0); assert( strcmp( "'\"' + \"' \\\"\"", "'\"' + \"' \\\"\"") == 0); assert( strcmp( "\"abc\"" , "\"abc\"") == 0); assert( strcmp( "x-y", "x-y") == 0); } void n_25( void) { int a = 1, b = 2, abc = 3, MACRO_0MACRO_1 = 2; assert( (a,b - 1) == 1); assert( ( - a) == -1); assert( abc == 3); assert( MACRO_0MACRO_1 == 2); assert( strcmp( "ZERO_TOKEN", "ZERO_TOKEN") == 0); } # 648 "n_std.c" int f( int a) { return a; } int g( int a) { return a * 2; } void n_26( void) { int x = 1; int AB = 1; int Z[1]; Z[0] = 1; assert( Z[0] == 1); assert( AB == 1); assert( x + f(x) == 2); assert( x + x + g( x) == 4); assert( Z[0] + f(Z[0]) == 2); } void n_27( void) { int a = 1, b = 2, c, m = 1, n = 2; # 711 "n_std.c" assert( 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 == 36); assert( (1) + (1 + 2) + 1 + 2 + 1 + 2 + 3 + 1 + 2 + 3 + 4 == 23); assert( 1 == 1); assert( ((a) - (b)) == -1); c = (a - b); assert( c == -1); assert( n == 2); } void n_28( void) { char * date = "Jan 13 2020"; assert( strcmp( "n_std.c", "n_std.c") == 0); assert( 751 == 779); assert( strlen( "Jan 13 2020") == 11); assert( date[ 4] != '0'); assert( strlen( "10:47:38") == 8); assert( 1); assert( __STDC_VERSION__ >= 199409L); # 1 "tests/test-c/line.h" 1 { char * file = "tests/test-c/line.h"; file += strlen( file) - 6; assert( 6 == 6 && strcmp( file, "line.h") == 0); } # 768 "n_std.c" 2 } void n_29( void) { int DEFINED = 1; assert( DEFINED == 1); } void n_30( void) { int a = 1, b = 2, c = 3; assert ( a + b + c == 6 ); } void n_32( void) { # 822 "n_std.c" } void n_37( void) { int ABCDEFGHIJKLMNOPQRSTUVWXYZabcde = 31; int ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ = 30; int nest = 0; assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcde == 31); assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ == 30); nest = 0; # 865 "n_std.c" nest = 8; # 874 "n_std.c" assert( nest == 8); nest = 0; # 1 "tests/test-c/nest1.h" 1 nest = 1; # 1 "tests/test-c/nest2.h" 1 nest = 2; # 1 "tests/test-c/nest3.h" 1 nest = 3; # 1 "tests/test-c/nest4.h" 1 nest = 4; # 1 "tests/test-c/nest5.h" 1 nest = 5; # 1 "tests/test-c/nest6.h" 1 nest = 6; # 1 "tests/test-c/nest7.h" 1 nest = 7; # 1 "tests/test-c/nest8.h" 1 nest = 8; # 5 "tests/test-c/nest7.h" 2 # 5 "tests/test-c/nest6.h" 2 # 5 "tests/test-c/nest5.h" 2 # 5 "tests/test-c/nest4.h" 2 # 5 "tests/test-c/nest3.h" 2 # 5 "tests/test-c/nest2.h" 2 # 5 "tests/test-c/nest1.h" 2 # 879 "n_std.c" 2 assert( nest == 8); nest = 32; assert( nest == 32); { char * extremely_long_string = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567" ; assert( strlen( extremely_long_string) == 507); } { int a123456789012345678901234567890 = 123450; int b123456789012345678901234567890 = 123451; int c123456789012345678901234567890 = 123452; int d123456789012345678901234567890 = 123453; int e123456789012345678901234567890 = 123454; int f123456789012345678901234567890 = 123455; int A123456789012345678901234567890 = 123456; int B123456789012345678901234567890 = 123457; int C123456789012345678901234567890 = 123458; int D1234567890123456789012 = 123459; assert( a123456789012345678901234567890 == 123450 && D1234567890123456789012 == 123459); } # 1 "tests/test-c/m1024.h" 1 # 927 "n_std.c" 2 assert( 1); } ================================================ FILE: tests/n_std-pcpp.i ================================================ # 30 "tests/test-c/n_std.c" void n_1( void); void n_2( void); void n_3( void); void n_4( void); void n_5( void); void n_6( void); void n_7( void); void n_9( void); void n_10( void); void n_11( void); void n_12( void); void n_13( void); void n_13_5( void); void n_13_7( void); void n_13_8( void); void n_13_13( void); void n_15( void); void n_18( void); void n_19( void); void n_20( void); void n_21( void); void n_22( void); void n_23( void); void n_24( void); void n_25( void); void n_26( void); void n_27( void); void n_28( void); void n_29( void); void n_30( void); void n_32( void); void n_37( void); int main( void) { n_2(); n_3(); n_5(); n_6(); n_7(); n_9(); n_10(); n_11(); n_12(); n_13(); n_13_5(); n_13_7(); n_13_8(); n_13_13(); n_15(); n_18(); n_19(); n_20(); n_21(); n_22(); n_23(); n_24(); n_25(); n_26(); n_27(); n_28(); n_29(); n_30(); n_32(); n_37(); puts( ""); return 0; } char quasi_trigraph[] = { '?', '?', ' ', '?', '?', '?', ' ' , '?', '?', '%', ' ', '?', '?', '^', ' ', '?', '#', '\0' }; void n_2( void) { int ab = 1, cd = 2, ef = 3, abcde = 5; assert( ab + cd + ef == 6); # 122 "tests/test-c/n_std.c" assert (ab + cd + ef == 6); assert (strcmp( "abcde", "abcde") == 0); assert( abcde == 5); } void n_3( void) { int abcd = 4; assert( strcmp( "abc de", "abc de") == 0); # 156 "tests/test-c/n_std.c" assert( abcd == 4); } void n_5( void) { int abcde = 5; assert( abcde == 5); } # 178 "tests/test-c/n_std.c" void n_6( void) { int abc = 3; assert( isalpha( 'a')); assert( abc == 3); assert( abc == 3); } void n_7( void) { #line 1234 "cpp" assert( 201 == 1234); assert( strcmp( "tests/test-c/n_std.c", "cpp") == 0); #line 2345 assert( 206 == 2345); assert( strcmp( "tests/test-c/n_std.c", "cpp") == 0); #line LINE_AND_FILENAME assert( 212 == 1234); assert( strcmp( "tests/test-c/n_std.c", "n_7.c") == 0); } #line 218 "n_std.c" void n_9( void) { #pragma who knows ? } void n_10( void) { # 238 "tests/test-c/n_std.c" assert( 1); # 251 "tests/test-c/n_std.c" } void n_11( void) { int abc = 1, a = 0; # 264 "tests/test-c/n_std.c" assert( abc); assert( abc); # 279 "tests/test-c/n_std.c" } void n_12( void) { fputs( "Bad evaluation of long.\n", stderr); fputs( "Bad evaluation of long.\n", stderr); # 325 "tests/test-c/n_std.c" } void n_13( void) # 343 "tests/test-c/n_std.c" { # 363 "tests/test-c/n_std.c" } void n_13_5( void) { # 387 "tests/test-c/n_std.c" } void n_13_7( void) { # 406 "tests/test-c/n_std.c" } void n_13_8( void) { # 439 "tests/test-c/n_std.c" } void n_13_13( void) { # 466 "tests/test-c/n_std.c" } void n_15( void) { assert( 1); # 483 "tests/test-c/n_std.c" assert( 1); } void n_18( void) { int c = 3; assert( (1-1) == 0); assert( ( c ) == 3); assert( strcmp( "n1:n2", "n1:n2") == 0); } void n_19( void) { int c = 1; # 521 "tests/test-c/n_std.c" assert( ( c ) == 1); } void n_20( void) { double fl; assert( sizeof fl == sizeof (double)); } void n_21( void) { int a = 1, x = 2, y = -3; assert( ---a == -1); assert( x--y == -1); } void n_22( void) { assert( strcmp( "12E+1", "12E+EXP") == 0); assert( strcmp( ".2e-1", ".2e-EXP") == 0); assert( strcmp( "12+1", "12+1") == 0); } void n_23( void) { int xy = 1; assert( xy == 1); assert( .12e+2 == 12.0); } void n_24( void) { assert( strcmp( "a+b", "a+b") == 0); assert( strcmp( "ab + cd", "ab + cd") == 0); assert( strcmp( "'\"' + \"' \\\"\"", "'\"' + \"' \\\"\"") == 0); assert( strcmp( "\"abc\"", "\"abc\"") == 0); assert( strcmp( "x-y", "x-y") == 0); } void n_25( void) { int a = 1, b = 2, abc = 3, MACRO_0MACRO_1 = 2; assert( (a,b - 1) == 1); assert( ( - a) == -1); assert( abc == 3); assert( MACRO_0MACRO_1 == 2); assert( strcmp( "ZERO_TOKEN", "ZERO_TOKEN") == 0); } # 648 "tests/test-c/n_std.c" int f( int a) { return a; } int g( int a) { return a * 2; } void n_26( void) { int x = 1; int AB = 1; int Z[1]; Z[0] = 1; assert( Z[0] == 1); assert( AB == 1); assert( x + f(x) == 2); assert( x + x + g( x) == 4); assert( Z[0] + f(Z[0]) == 2); } void n_27( void) { int a = 1, b = 2, c, m = 1, n = 2; # 711 "tests/test-c/n_std.c" assert( 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 == 36); assert( (1) + (1 + 2) + 1 + 2 + 1 + 2 + 3 + 1 + 2 + 3 + 4 == 23); assert( 1 == 1); assert( ((a) - (b)) == -1); c = (a - b); assert( c == -1); assert( n == 2); } void n_28( void) { char * date = "Jan 13 2020"; assert( strcmp( "tests/test-c/n_std.c", "n_std.c") == 0); assert( 731 == 751); assert( strlen( "Jan 13 2020") == 11); assert( date[ 4] != '0'); assert( strlen( "10:47:38") == 8); assert( 1); assert( 199901L >= 199409L); # 3 "tests/test-c/line.h" { char * file = "tests/test-c/line.h"; file += strlen( file) - 6; assert( 731 == 6 && strcmp( file, "line.h") == 0); } # 768 "tests/test-c/n_std.c" } void n_29( void) { int DEFINED = 1; assert( DEFINED == 1); } void n_30( void) { int a = 1, b = 2, c = 3; assert ( a + b + c == 6 ); } void n_32( void) { # 822 "tests/test-c/n_std.c" } void n_37( void) { int ABCDEFGHIJKLMNOPQRSTUVWXYZabcde = 31; int ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ = 30; int nest = 0; assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcde == 31); assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ == 30); nest = 0; # 865 "tests/test-c/n_std.c" nest = 8; # 874 "tests/test-c/n_std.c" assert( nest == 8); nest = 0; # 3 "tests/test-c/nest1.h" nest = 1; # 3 "tests/test-c/nest2.h" nest = 2; # 3 "tests/test-c/nest3.h" nest = 3; # 3 "tests/test-c/nest4.h" nest = 4; # 3 "tests/test-c/nest5.h" nest = 5; # 3 "tests/test-c/nest6.h" nest = 6; # 3 "tests/test-c/nest7.h" nest = 7; # 4 "tests/test-c/nest8.h" nest = 8; # 879 "tests/test-c/n_std.c" assert( nest == 8); nest = 32; assert( nest == 32); { char * extremely_long_string = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567" # 901 "tests/test-c/n_std.c" ; assert( strlen( extremely_long_string) == 507); } { int a123456789012345678901234567890 = 123450; int b123456789012345678901234567890 = 123451; int c123456789012345678901234567890 = 123452; int d123456789012345678901234567890 = 123453; int e123456789012345678901234567890 = 123454; int f123456789012345678901234567890 = 123455; int A123456789012345678901234567890 = 123456; int B123456789012345678901234567890 = 123457; int C123456789012345678901234567890 = 123458; int D1234567890123456789012 = 123459; # 917 "tests/test-c/n_std.c" assert( a123456789012345678901234567890 == 123450 && D1234567890123456789012 == 123459); } # 927 "tests/test-c/n_std.c" assert( 1); } ================================================ FILE: tests/n_std.i ================================================ # 30 "tests/test-c/n_std.c" void n_1( void); void n_2( void); void n_3( void); void n_4( void); void n_5( void); void n_6( void); void n_7( void); void n_9( void); void n_10( void); void n_11( void); void n_12( void); void n_13( void); void n_13_5( void); void n_13_7( void); void n_13_8( void); void n_13_13( void); void n_15( void); void n_18( void); void n_19( void); void n_20( void); void n_21( void); void n_22( void); void n_23( void); void n_24( void); void n_25( void); void n_26( void); void n_27( void); void n_28( void); void n_29( void); void n_30( void); void n_32( void); void n_37( void); int main( void) { n_2(); n_3(); n_5(); n_6(); n_7(); n_9(); n_10(); n_11(); n_12(); n_13(); n_13_5(); n_13_7(); n_13_8(); n_13_13(); n_15(); n_18(); n_19(); n_20(); n_21(); n_22(); n_23(); n_24(); n_25(); n_26(); n_27(); n_28(); n_29(); n_30(); n_32(); n_37(); puts( ""); return 0; } char quasi_trigraph[] = { '?', '?', ' ', '?', '?', '?', ' ' , '?', '?', '%', ' ', '?', '?', '^', ' ', '?', '#', '\0' }; void n_2( void) { int ab = 1, cd = 2, ef = 3, abcde = 5; assert( ab + cd + ef == 6); # 122 "tests/test-c/n_std.c" assert (ab + cd + ef == 6); assert (strcmp( "abcde", "abcde") == 0); assert( abcde == 5); } void n_3( void) { int abcd = 4; assert( strcmp( "abc de", "abc de") == 0); # 156 "tests/test-c/n_std.c" assert( abcd == 4); } void n_5( void) { int abcde = 5; assert( abcde == 5); } # 178 "tests/test-c/n_std.c" void n_6( void) { int abc = 3; assert( isalpha( 'a')); assert( abc == 3); assert( abc == 3); } void n_7( void) { #line 1234 "cpp" assert( 201 == 1234); assert( strcmp( "tests/test-c/n_std.c", "cpp") == 0); #line 2345 assert( 206 == 2345); assert( strcmp( "tests/test-c/n_std.c", "cpp") == 0); #line LINE_AND_FILENAME assert( 212 == 1234); assert( strcmp( "tests/test-c/n_std.c", "n_7.c") == 0); } #line 218 "n_std.c" void n_9( void) { #pragma who knows ? } void n_10( void) { # 238 "tests/test-c/n_std.c" assert( 1); # 251 "tests/test-c/n_std.c" } void n_11( void) { int abc = 1, a = 0; # 264 "tests/test-c/n_std.c" assert( abc); assert( abc); # 279 "tests/test-c/n_std.c" } void n_12( void) { fputs( "Bad evaluation of long.\n", stderr); fputs( "Bad evaluation of long.\n", stderr); # 325 "tests/test-c/n_std.c" } void n_13( void) # 343 "tests/test-c/n_std.c" { # 363 "tests/test-c/n_std.c" } void n_13_5( void) { # 387 "tests/test-c/n_std.c" } void n_13_7( void) { # 406 "tests/test-c/n_std.c" } void n_13_8( void) { # 439 "tests/test-c/n_std.c" } void n_13_13( void) { # 466 "tests/test-c/n_std.c" } void n_15( void) { assert( 1); # 483 "tests/test-c/n_std.c" assert( 1); } void n_18( void) { int c = 3; assert( (1-1) == 0); assert( ( c ) == 3); assert( strcmp( "n1:n2", "n1:n2") == 0); } void n_19( void) { int c = 1; # 521 "tests/test-c/n_std.c" assert( ( c ) == 1); } void n_20( void) { double fl; assert( sizeof fl == sizeof (double)); } void n_21( void) { int a = 1, x = 2, y = -3; assert( ---a == -1); assert( x--y == -1); } void n_22( void) { assert( strcmp( "12E+1", "12E+EXP") == 0); assert( strcmp( ".2e-1", ".2e-EXP") == 0); assert( strcmp( "12+1", "12+1") == 0); } void n_23( void) { int xy = 1; assert( xy == 1); assert( .12e+2 == 12.0); } void n_24( void) { assert( strcmp( "a+b", "a+b") == 0); assert( strcmp( "ab + cd", "ab + cd") == 0); assert( strcmp( "'\"' + \"' \\\"\"", "'\"' + \"' \\\"\"") == 0); assert( strcmp( "\"abc\"", "\"abc\"") == 0); assert( strcmp( "x-y", "x-y") == 0); } void n_25( void) { int a = 1, b = 2, abc = 3, MACRO_0MACRO_1 = 2; assert( (a,b - 1) == 1); assert( ( - a) == -1); assert( abc == 3); assert( MACRO_0MACRO_1 == 2); assert( strcmp( "ZERO_TOKEN", "ZERO_TOKEN") == 0); } # 648 "tests/test-c/n_std.c" int f( int a) { return a; } int g( int a) { return a * 2; } void n_26( void) { int x = 1; int AB = 1; int Z[1]; Z[0] = 1; assert( Z[0] == 1); assert( AB == 1); assert( x + f(x) == 2); assert( x + x + g( x) == 4); assert( Z[0] + f(Z[0]) == 2); } void n_27( void) { int a = 1, b = 2, c, m = 1, n = 2; # 711 "tests/test-c/n_std.c" assert( 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 == 36); assert( (1) + (1 + 2) + 1 + 2 + 1 + 2 + 3 + 1 + 2 + 3 + 4 == 23); assert( 1 == 1); assert( ((a) - (b)) == -1); c = (a - b); assert( c == -1); assert( n == 2); } void n_28( void) { char * date = "Jan 13 2020"; assert( strcmp( "tests/test-c/n_std.c", "n_std.c") == 0); assert( 731 == 751); assert( strlen( "Jan 13 2020") == 11); assert( date[ 4] != '0'); assert( strlen( "10:47:38") == 8); assert( 1); assert( 199901L >= 199409L); # 3 "tests/test-c/line.h" { char * file = "tests/test-c/line.h"; file += strlen( file) - 6; assert( 731 == 6 && strcmp( file, "line.h") == 0); } # 768 "tests/test-c/n_std.c" } void n_29( void) { int DEFINED = 1; assert( DEFINED == 1); } void n_30( void) { int a = 1, b = 2, c = 3; assert ( a + b + c == 6 ); } void n_32( void) { # 822 "tests/test-c/n_std.c" } void n_37( void) { int ABCDEFGHIJKLMNOPQRSTUVWXYZabcde = 31; int ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ = 30; int nest = 0; assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcde == 31); assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ == 30); nest = 0; # 865 "tests/test-c/n_std.c" nest = 8; # 874 "tests/test-c/n_std.c" assert( nest == 8); nest = 0; # 3 "tests/test-c/nest1.h" nest = 1; # 3 "tests/test-c/nest2.h" nest = 2; # 3 "tests/test-c/nest3.h" nest = 3; # 3 "tests/test-c/nest4.h" nest = 4; # 3 "tests/test-c/nest5.h" nest = 5; # 3 "tests/test-c/nest6.h" nest = 6; # 3 "tests/test-c/nest7.h" nest = 7; # 4 "tests/test-c/nest8.h" nest = 8; # 879 "tests/test-c/n_std.c" assert( nest == 8); nest = 32; assert( nest == 32); { char * extremely_long_string = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567" # 901 "tests/test-c/n_std.c" ; assert( strlen( extremely_long_string) == 507); } { int a123456789012345678901234567890 = 123450; int b123456789012345678901234567890 = 123451; int c123456789012345678901234567890 = 123452; int d123456789012345678901234567890 = 123453; int e123456789012345678901234567890 = 123454; int f123456789012345678901234567890 = 123455; int A123456789012345678901234567890 = 123456; int B123456789012345678901234567890 = 123457; int C123456789012345678901234567890 = 123458; int D1234567890123456789012 = 123459; # 917 "tests/test-c/n_std.c" assert( a123456789012345678901234567890 == 123450 && D1234567890123456789012 == 123459); } # 927 "tests/test-c/n_std.c" assert( 1); } ================================================ FILE: tests/n_std.py ================================================ import unittest, time, difflib clock = time.process_time class n_std(unittest.TestCase): def runTest(self): from pcpp import Preprocessor import os start = clock() p = Preprocessor() p.compress = 1 p.line_directive = '#' p.define('__STDC__ 1') p.define('__STDC_VERSION__ 199901L') p.define('__DATE__ "Jan 13 2020"') p.define('__TIME__ "10:47:38"') p.define('NO_SYSTEM_HEADERS') path = 'tests/test-c/n_std.c' with open(path, 'rt') as ih: p.parse(ih.read(), path) with open('tests/n_std.i', 'w') as oh: p.write(oh) end = clock() print("Preprocessed", path, "in", end-start, "seconds") self.assertEqual(p.return_code, 0) with open('tests/n_std.i', 'rt') as ih: written = ih.readlines() with open('tests/n_std-pcpp.i', 'rt') as ih: reference = ih.readlines() if written != reference: print("pcpp is not emitting its reference output! Differences:") for line in difflib.unified_diff(reference, written, fromfile='n_std-pcpp.i', tofile='n_std.i'): print(line, end='') self.assertTrue(False) ================================================ FILE: tests/passthru.py ================================================ import unittest, time from io import StringIO clock = time.process_time class runner(object): def runTest(self): from pcpp import Preprocessor, OutputDirective, Action import os, sys class PassThruPreprocessor(Preprocessor): def on_include_not_found(self,is_malformed,is_system_include,curdir,includepath): raise OutputDirective(Action.IgnoreAndPassThrough) def on_unknown_macro_in_defined_expr(self,tok): return None # Pass through as expanded as possible def on_unknown_macro_in_expr(self,ident): return None # Pass through as expanded as possible def on_unknown_macro_function_in_expr(self,ident): return None # Pass through as expanded as possible def on_directive_handle(self,directive,toks,ifpassthru,precedingtoks): super(PassThruPreprocessor, self).on_directive_handle(directive,toks,ifpassthru,precedingtoks) return None # Pass through where possible def on_directive_unknown(self,directive,toks,ifpassthru,precedingtoks): if directive.value == 'error' or directive.value == 'warning': super(PassThruPreprocessor, self).on_directive_unknown(directive,toks,ifpassthru,precedingtoks) # Pass through raise OutputDirective(Action.IgnoreAndPassThrough) def on_comment(self,tok): # Pass through return True start = clock() p = PassThruPreprocessor() p.passthru_expr_has_include = True p.parse(self.input) oh = StringIO() p.write(oh) end = clock() print("Preprocessed test in", end-start, "seconds") if oh.getvalue() != self.output: print("Should be:\n" + self.output + "EOF\n", file = sys.stderr) print("\nWas:\n" + oh.getvalue()+"EOF\n", file = sys.stderr) self.assertEqual(p.return_code, 0) self.assertEqual(oh.getvalue(), self.output) class test1(unittest.TestCase, runner): input = r"""#if 5 I am five #else I am not five #endif""" output = r""" I am five """ class test2(unittest.TestCase, runner): input = r"""#if UNKNOWN I am five #else I am not five #endif""" output = r"""#if UNKNOWN I am five #else I am not five #endif """ class test3(unittest.TestCase, runner): input = r"""#if UNKNOWN A #elif ALSO_UNKNOWN B #else C #endif""" output = r"""#if UNKNOWN A #elif ALSO_UNKNOWN B #else C #endif """ class test4(unittest.TestCase, runner): input = r"""#define ALSO_UNKNOWN 1 #if UNKNOWN A #elif ALSO_UNKNOWN B #else C #endif""" output = r"""#define ALSO_UNKNOWN 1 #if UNKNOWN A #elif 1 B #else C #endif """ class test5(unittest.TestCase, runner): input = r"""#define ALSO_UNKNOWN 0 #if UNKNOWN A #elif ALSO_UNKNOWN B #else C #endif""" output = r"""#define ALSO_UNKNOWN 0 #if UNKNOWN A #else C #endif """ class test6(unittest.TestCase, runner): input = r"""#define UNKNOWN 1 #if UNKNOWN A #elif ALSO_UNKNOWN B #else C #endif""" output = r"""#define UNKNOWN 1 A """ class test7(unittest.TestCase, runner): input = r"""#define UNKNOWN 0 #if UNKNOWN A #elif ALSO_UNKNOWN B #else C #endif""" output = r"""#define UNKNOWN 0 #if ALSO_UNKNOWN B #else C #endif """ class test8(unittest.TestCase, runner): input = r"""#define UNKNOWN 0 #if UNKNOWN #if 1 A #else AA #endif #elif ALSO_UNKNOWN #if 1 B #else BB #endif #else #if 1 C #else CC #endif #endif""" output = r"""#define UNKNOWN 0 #if ALSO_UNKNOWN B #else C #endif """ class test9(unittest.TestCase, runner): input = r"""#define KNOWN 0 #if defined(UNKNOWN) || KNOWN A #endif """ output = r"""#define KNOWN 0 #if defined(UNKNOWN) || 0 A #endif """ class test10(unittest.TestCase, runner): input = r"""#if !defined(__cpp_constexpr) #if __cplusplus >= 201402L #define __cpp_constexpr 201304 // relaxed constexpr #else #define __cpp_constexpr 190000 #endif #endif """ output = r"""#if !defined(__cpp_constexpr) #if __cplusplus >= 201402L #define __cpp_constexpr 201304 // relaxed constexpr #else #define __cpp_constexpr 190000 #endif #endif """ class test11(unittest.TestCase, runner): input = r"""#define __cpp_constexpr 201304 #if !defined(__cpp_constexpr) #if __cplusplus >= 201402L #define __cpp_constexpr 201304 // relaxed constexpr #else #define __cpp_constexpr 190000 #endif #endif #ifndef BOOSTLITE_CONSTEXPR #if __cpp_constexpr >= 201304 #define BOOSTLITE_CONSTEXPR constexpr #endif #endif #ifndef BOOSTLITE_CONSTEXPR #define BOOSTLITE_CONSTEXPR #endif """ output = r"""#define __cpp_constexpr 201304 #line 9 #ifndef BOOSTLITE_CONSTEXPR #define BOOSTLITE_CONSTEXPR constexpr #endif """ class test12(unittest.TestCase, runner): input = r""" #define BOOST_OUTCOME_DISABLE_PREPROCESSED_INTERFACE_FILE #ifndef BOOST_OUTCOME_DISABLE_PREPROCESSED_INTERFACE_FILE #else #if defined(_MSC_VER) && !defined(__clang__) #define BOOST_OUTCOME_HEADERS_PATH2 BOOST_OUTCOME_VERSION_GLUE(v, BOOST_OUTCOME_HEADERS_VERSION, /monad.hpp) #elif 1 #define BOOST_OUTCOME_HEADERS_PATH2 BOOST_OUTCOME_VERSION_GLUE(v, BOOST_OUTCOME_HEADERS_VERSION,)/monad.hpp #endif #endif """ output = r""" #define BOOST_OUTCOME_DISABLE_PREPROCESSED_INTERFACE_FILE #if defined(_MSC_VER) && !defined(__clang__) #define BOOST_OUTCOME_HEADERS_PATH2 BOOST_OUTCOME_VERSION_GLUE(v, BOOST_OUTCOME_HEADERS_VERSION, /monad.hpp) #elif 1 #define BOOST_OUTCOME_HEADERS_PATH2 BOOST_OUTCOME_VERSION_GLUE(v, BOOST_OUTCOME_HEADERS_VERSION,)/monad.hpp #endif """ class test18(unittest.TestCase, runner): input = r""" /* multiline comment */ void shouldBeOnLineSeven(); """ output = r""" /* multiline comment */ void shouldBeOnLineSeven(); """ class test19(unittest.TestCase, runner): input = r""" /* a comment that spans eight lines */ void shouldBeOnLineEleven();""" output = r""" /* a comment that spans eight lines */ void shouldBeOnLineEleven(); """ class test20(unittest.TestCase, runner): input = r""" #include ASIO_CUSTOM_HANDLER_TRACKING """ output = r""" #include ASIO_CUSTOM_HANDLER_TRACKING """ class test21(unittest.TestCase, runner): input = r""" #if !FOO(5) hi #endif """ output = r""" #if !FOO(5) hi #endif """ class test22(unittest.TestCase, runner): input = r""" #if !__has_include() hi #endif """ output = r""" #if !__has_include() hi #endif """ class test23(unittest.TestCase, runner): input = r""" #if 0 // Do NOT enable weakened implicit construction for these types hi #endif """ output = r"""""" if __name__ == '__main__': unittest.main() ================================================ FILE: tests/test-c/LICENSE ================================================ /*- * Copyright (c) 1998, 2002-2008 Kiyoshi Matsui * All rights reserved. * * This software including the files in this directory is provided under * the following license. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ ================================================ FILE: tests/test-c/defs.h ================================================ /* defs.h */ #ifndef NO_SYSTEM_HEADERS #include /* assert(): Enable one of these three. */ /* Note: This source doesn't use #elif directive to test preprocessor which can't recognize the directive. */ #if 1 /* For the translator which can process properly. */ #include #else #if 0 /* Not to abort on error. */ #define assert( exp) (exp) ? (void)0 : (void) fprintf( stderr, \ "Assertion failed: %s, from line %d of file %s\n", \ # exp, __LINE__, __FILE__) #endif #endif extern int strcmp( const char *, const char *); extern size_t strlen( const char *); extern void exit( int); #endif ================================================ FILE: tests/test-c/e_12_8.c ================================================ /* e_12_8.c: Out of range of integer pp-token in #if expression. */ /* 12.8: Preprocessing number perhaps out of range of unsigned long. */ #if 123456789012345678901 #endif main( void) { return 0; } ================================================ FILE: tests/test-c/e_14.c ================================================ /* e_14.c: Illegal #if expressions. */ #define A 1 #define B 1 /* 14.1: String literal is not allowed in #if expression. */ #if "string" #endif /* The second error ? */ /* 14.2: Operators =, +=, ++, etc. are not allowed in #if expression. */ #if A = B #endif #if A++ B #endif #if A --B #endif #if A.B #endif /* 14.3: Unterminated #if expression. */ #if 0 < #endif #if ( (A == B) #endif /* 14.4: Unbalanced parenthesis in #if defined operator. */ #if defined ( MACRO #endif /* 14.5: No argument. */ #if #endif /* 14.6: Macro expanding to 0 token in #if expression. */ #define ZERO_TOKEN #if ZERO_TOKEN #endif main( void) { return 0; } ================================================ FILE: tests/test-c/e_14_10.c ================================================ /* e_14_10.c: Overflow of constant expression in #if directive. */ /* 14.10: */ #include #if LONG_MAX - LONG_MIN #endif #if LONG_MAX + 1 #endif #if LONG_MIN - 1 #endif #if LONG_MAX * 2 #endif int main( void) { return 0; } ================================================ FILE: tests/test-c/e_14_7.c ================================================ /* e_14_7.c: There is no keyword in #if expression. */ /* 14.7: sizeof operator is disallowed. */ /* Evaluated as: 0 (0) Constant expression syntax error. */ #if sizeof (int) #endif /* 14.8: type cast is disallowed. */ /* Evaluated as: (0)0x8000 Also a constant expression error. */ #if (int)0x8000 < 0 #endif main( void) { return 0; } ================================================ FILE: tests/test-c/e_14_9.c ================================================ /* e_14_9.c: Out of range in #if expression (division by 0). */ /* 14.9: Divided by 0. */ #if 1 / 0 #endif int main( void) { return 0; } ================================================ FILE: tests/test-c/e_15_3.c ================================================ /* e_15_3.c: #ifdef, #ifndef syntax errors. */ /* 15.3: Not an identifier. */ #ifdef "string" #endif #ifdef 123 #endif /* 15.4: Excessive token sequence. */ #ifdef MACRO Junk #endif /* 15.5: No argument. */ #ifndef #endif main( void) { return 0; } ================================================ FILE: tests/test-c/e_16.c ================================================ /* e_16.c: Trailing junk of #else, #endif. */ /* 16.1: Trailing junk of #else. */ #define MACRO_0 0 #if MACRO_0 #else MACRO_0 /* 16.2: Trailing junk of #endif. */ #endif MACRO_0 main( void) { return 0; } ================================================ FILE: tests/test-c/e_17.c ================================================ /* e_17.c: Ill-formed group in a source file. */ #define MACRO_1 1 /* 17.1: Error of #endif without #if. */ #endif /* 17.2: Error of #else without #if. */ #else /* 17.3: Error of #else after #else. */ #if MACRO_1 #else #else #endif /* 17.4: Error of #elif after #else. */ #if MACRO_1 == 1 #else #elif MACRO_1 == 0 #endif /* 17.5: Error of #endif without #if in an included file. */ #if 1 #include "unbal1.h" /* 17.6: Error of unterminated #if section in an included file. */ #include "unbal2.h" #endif /* 17.7: Error of unterminated #if section. */ #if MACRO_1 == 0 #else ================================================ FILE: tests/test-c/e_18_4.c ================================================ /* e_18_4.c: #define syntax errors. */ /* 18.4: Not an identifier. */ #define "string" #define 123 /* 18.5: No argument. */ #define /* 18.6: Empty parameter list. */ #define math( op, a, ) op( (a), (b)) /* 18.7: Duplicate parameter names. */ #define math( op, a, a) op( (a), (b)) /* 18.8: Argument is not an identifier. */ #define NUMARGS( 1, +, 2) (1 + 2) /* 18.9: No space between macro name and replacement text. */ /* C90 (Corrigendum 1) forbids this if and only the replacement text begins with a non-basic-character. C99 forbids this even when the replacement text begins with basic- character. */ /* From ISO 9899:1990 / Corrigendum 1. */ #define THIS$AND$THAT(a, b) ((a) + (b)) /* Note: the following definition is legal (object-like macro). #define THIS $AND$THAT(a, b) ((a) + (b)) */ main( void) { return 0; } ================================================ FILE: tests/test-c/e_19_3.c ================================================ /* e_19_3.c: Redefinitions of macros. */ #include "defs.h" #define str( s) # s #define xstr( s) str( s) /* Excerpts from ISO C 6.8.3 "Examples". */ #define OBJ_LIKE (1-1) #define FTN_LIKE(a) ( a ) /* The following redefinitions should be diagnosed. */ /* 19.3: */ #define OBJ_LIKE (0) /* different token sequence */ /* 19.4: */ #undef OBJ_LIKE #define OBJ_LIKE (1-1) #define OBJ_LIKE (1 - 1) /* different white space */ /* 19.5: */ #define FTN_LIKE(b) ( a ) /* different parameter usage */ /* 19.6: */ #undef FTN_LIKE #define FTN_LIKE(a) ( a ) #define FTN_LIKE(b) ( b ) /* different parameter spelling */ /* 19.7: Not in ISO C "Examples" */ #define FTN_LIKE OBJ_LIKE main( void) { return 0; } ================================================ FILE: tests/test-c/e_23_3.c ================================================ /* e_23_3.c: ## operator shall not occur at the beginning or at the end of replacement list for either form of macro definition. */ /* 23.3: In object-like macro. */ #define con ## name #define cat 12 ## /* 23.4: In function-like macro. */ #define CON( a, b) ## a ## b #define CAT( b, c) b ## c ## main( void) { return 0; } ================================================ FILE: tests/test-c/e_24_6.c ================================================ /* e_24_6.c: Operand of # operator in function-like macro definition should be a parameter. */ /* 24.6: */ #define FUNC( a) # b main( void) { return 0; } ================================================ FILE: tests/test-c/e_25_6.c ================================================ /* e_25_6.c: Macro arguments are pre-expanded separately. */ /* 25.6: */ #define sub( x, y) (x - y) #define head sub( #define body(x,y) x,y #define tail ) #define head_body_tail( a, b, c) a b c /* "head" is once expanded to "sub(", then rescanning of "sub(" causes an uncompleted macro call. Expansion of an argument should complete within the argument. */ head_body_tail( head, body(a,b), tail); main( void) { return 0; } ================================================ FILE: tests/test-c/e_27_7.c ================================================ /* e_27_7.c: Error of rescanning. */ #define sub( x, y) (x - y) /* 27.7: */ #define TWO_TOKENS a,b #define SUB( x, y) sub( x, y) /* Too many arguments error while rescanning after once replaced to: sub( a,b, 1); */ SUB( TWO_TOKENS, 1); main( void) { return 0; } ================================================ FILE: tests/test-c/e_29_3.c ================================================ /* e_29_3.c: #undef errors. */ /* 29.3: Not an identifier. */ #undef "string" #undef 123 /* 29.4: Excessive token sequence. */ #undef MACRO_0 Junk /* 29.5: No argument. */ #undef main( void) { return 0; } ================================================ FILE: tests/test-c/e_31.c ================================================ /* e_31.c: Illegal macro calls. */ #define sub( x, y) (x - y) /* 31.1: Too many arguments error. */ sub( x, y, z); /* 31.2: Too few arguments error. */ sub( x); main( void) { return 0; } ================================================ FILE: tests/test-c/e_31_3.c ================================================ /* e_31_3.c: Macro call in control line should complete in the line. */ #define glue( a, b) a ## b #define str( s) # s #define xstr( s) str( s) /* 31.3: Unterminated macro call. */ #include xstr( glue( header, .h)) main( void) { return 0; } ================================================ FILE: tests/test-c/e_32_5.c ================================================ /* e_32_5.c: Range error of character constant. */ /* 32.5: Value of a numerical escape sequence in character constant should be in the range of char. */ #if '\x123' == 0x123 /* Out of range. */ #endif main( void) { return 0; } ================================================ FILE: tests/test-c/e_33_2.c ================================================ /* e_33_2.c: Out of range of numerical escape sequence in wide-char. */ /* 33.2: Value of a numerical escape sequence in wide-character constant should be in the range of wchar_t. */ #if L'\xabcdef012' == 0xbcdef012 /* Perhaps out of range. */ #endif main( void) { return 0; } ================================================ FILE: tests/test-c/e_35_2.c ================================================ /* e_35_2.c: Out of range of character constant. */ /* In ASCII character set. */ /* 35.2: */ #if 'abcdefghi' == '\x61\x62\x63\x64\x65\x66\x67\x68\x69' /* Perhaps out of range. */ #endif main( void) { return 0; } ================================================ FILE: tests/test-c/e_4_3.c ================================================ /* e_4_3.c: Illegal pp-token. */ /* 4.3: Empty character constant is an error. */ #if '' == 0 /* This line is invalid, maybe skipped. */ #endif /* This line maybe the second error. */ main( void) { return 0; } ================================================ FILE: tests/test-c/e_7_4.c ================================================ /* e_7_4.c: #line error. */ /* 7.4: string literal in #line directive shall be a character string literal. */ #line 123 L"wide" main( void) { return 0; } ================================================ FILE: tests/test-c/e_std.c ================================================ /* * e_std.c * * 1998/08 made public kmatsui * 2002/08 revised not to conflict with C99 Standard kmatsui * 2003/11 added a few samples kmatsui * * Samples to test Standard C preprocessing. * Preprocessor must diagnose all of these samples appropriately. */ void e_19_3( void); void e_25_6( void); void e_27_6( void); void e_31( void); main( void) { e_19_3(); e_25_6(); e_27_6(); e_31(); return 0; } /* Illegal pp-token. */ /* 4.3: Empty character constant is an error. */ #if '' == 0 /* This line is invalid, maybe skipped. */ #endif /* This line maybe the second error. */ /* #line error. */ /* 7.4: string literal in #line directive shall be a character string literal. */ #line 123 L"wide" /* Out of range of integer pp-token in #if expression. */ /* Note: Tests of character constant overflow are in 32.5, 33.2, 35.2. */ /* 12.8: Preprocessing number perhaps out of range of unsigned long. */ #if 123456789012345678901 #endif /* Illegal #if expressions. */ #define A 1 #define B 1 /* 14.1: String literal is not allowed in #if expression. */ #if "string" #endif /* The second error ? */ /* 14.2: Operators =, +=, ++, etc. are not allowed in #if expression. */ #if A = B #endif #if A++ B #endif #if A --B #endif #if A.B #endif /* 14.3: Unterminated #if expression. */ #if 0 < #endif #if ( (A == B) #endif /* 14.4: Unbalanced parenthesis in #if defined operator. */ #if defined ( MACRO #endif /* 14.5: No argument. */ #if #endif /* 14.6: Macro expanding to 0 token in #if expression. */ #define ZERO_TOKEN #if ZERO_TOKEN #endif /* There is no keyword in #if expression. */ /* 14.7: sizeof operator is disallowed. */ /* Evaluated as: 0 (0) Constant expression syntax error. */ #if sizeof (int) #endif /* 14.8: type cast is disallowed. */ /* Evaluated as: (0)0x8000 Also a constant expression error. */ #if (int)0x8000 < 0 #endif /* Out of range in #if expression (division by 0). */ /* 14.9: Divided by 0. */ #if 1 / 0 #endif /* Overflow of constant expression in #if directive. */ /* 14.10: */ #include #if LONG_MAX - LONG_MIN #endif #if LONG_MAX + 1 #endif #if LONG_MIN - 1 #endif #if LONG_MAX * 2 #endif /* #ifdef, #ifndef syntax errors. */ /* 15.3: Not an identifier. */ #ifdef "string" #endif #ifdef 123 #endif /* 15.4: Excessive token sequence. */ #ifdef A Junk #endif /* 15.5: No argument. */ #ifndef #endif /* Trailing junk of #else, #endif. */ /* 16.1: Trailing junk of #else. */ #define MACRO_0 0 #if MACRO_0 #else MACRO_0 /* 16.2: Trailing junk of #endif. */ #endif MACRO_0 /* Ill-formed group in a source file. */ #define MACRO_1 1 /* 17.1: Error of #endif without #if. */ #endif /* 17.2: Error of #else without #if. */ #else /* 17.3: Error of #else after #else. */ #if MACRO_1 #else /* line 168 */ #if 1 #else #endif #else #endif /* 17.4: Error of #elif after #else. */ #if MACRO_1 == 1 #else /* line 177 */ #elif MACRO_1 == 0 #endif /* 17.5: Error of #endif without #if in an included file. */ #if 1 #include "unbal1.h" /* 17.6: Error of unterminated #if section in an included file. */ #include "unbal2.h" #endif /* 17.7: Error of unterminated #if section. */ /* An error would be reported at end of file. */ #if MACRO_1 == 0 /* line 191 */ #else /* #define syntax errors. */ /* 18.4: Not an identifier. */ #define "string" #define 123 /* 18.5: No argument. */ #define /* 18.6: Empty parameter list. */ #define math( op, a, ) op( (a), (b)) /* 18.7: Duplicate parameter names. */ #define math( op, a, a) op( (a), (b)) /* 18.8: Parameter is not an identifier. */ #define NUMARGS( 1, +, 2) (1 + 2) /* 18.9: No space between macro name and replacement text. */ /* C90 (Corrigendum 1) forbids this if and only the replacement text begins with a non-basic-character. C99 forbids this even when the replacement text begins with basic- character. */ /* From ISO 9899:1990 / Corrigendum 1. */ #define THIS$AND$THAT(a, b) ((a) + (b)) /* Note: the following definition is legal (object-like macro). #define THIS $AND$THAT(a, b) ((a) + (b)) */ /* Redefinitions of macros. */ #define str( s) # s #define xstr( s) str( s) /* Excerpts from ISO C 6.8.3 "Examples". */ #define OBJ_LIKE (1-1) #define FTN_LIKE(a) ( a ) void e_19_3( void) { /* The following redefinitions should be diagnosed. */ /* 19.3: */ #define OBJ_LIKE (0) /* different token sequence */ /* 19.4: */ #undef OBJ_LIKE #define OBJ_LIKE (1-1) #define OBJ_LIKE (1 - 1) /* different white space */ /* 19.5: */ #define FTN_LIKE(b) ( a ) /* different parameter usage */ /* 19.6: */ #undef FTN_LIKE #define FTN_LIKE(a) ( a ) #define FTN_LIKE(b) ( b ) /* different parameter spelling */ /* 19.7: Not in ISO C "Examples" */ #define FTN_LIKE OBJ_LIKE } /* ## operator shall not occur at the beginning or at the end of replacement list for either form of macro definition. */ /* 23.3: In object-like macro. */ #define con ## name #define cat 12 ## /* 23.4: In function-like macro. */ #define CON( a, b) ## a ## b #define CAT( b, c) b ## c ## /* Operand of # operator in function-like macro definition shall be a parameter name. */ /* 24.6: */ #define FUNC( a) # b /* Macro arguments are pre-expanded separately. */ /* 25.6: */ #define sub( x, y) (x - y) #define head sub( #define body(x,y) x,y #define tail ) #define head_body_tail( a, b, c) a b c void e_25_6( void) { /* "head" is once replaced to "sub(", then rescanning of "sub(" causes an uncompleted macro call. Expansion of an argument should complete within the argument. */ head_body_tail( head, body(a,b), tail); } /* Error of rescanning. */ /* 27.7: */ #define TWO_ARGS a,b #define SUB( x, y) sub( x, y) void e_27_7( void) { /* Too many arguments error while rescanning after once replaced to: sub( a,b, 1); */ SUB( TWO_ARGS, 1); } /* #undef errors. */ /* 29.3: Not an identifier. */ #undef "string" #undef 123 /* 29.4: Excessive token sequence. */ #undef MACRO_0 Junk /* 29.5: No argument. */ #undef /* Illegal macro calls. */ void e_31( void) { int x = 1, y = 2; /* 31.1: Too many arguments error. */ sub( x, y, z); /* 31.2: Too few arguments error. */ sub( x); } /* Macro call in control line should complete in the line. */ #define glue( a, b) a ## b /* 31.3: Unterminated macro call. */ #include xstr( glue( header, .h)) /* Range error of character constant. */ /* 32.5: Value of a numerical escape sequence in character constant should be in the range of char. */ #if '\x123' == 0x123 /* Out of range. */ #endif /* Out of range of numerical escape sequence in wide-char. */ /* 33.2: Value of a numerical escape sequence in wide-character constant should be in the range of wchar_t. */ #if L'\xabcdef012' == 0xbcde012 /* Perhaps out of range. */ #endif /* Out of range of character constant. */ /* In ASCII character set. */ /* 35.2: */ #if 'abcdefghi' == '\x61\x62\x63\x64\x65\x66\x67\x68\x69' /* Perhaps out of range. */ #endif /* Error of "unterminated #if section started at line 191" will be reported at end of file. */ ================================================ FILE: tests/test-c/header.h ================================================ /* header.h */ #define MACRO_abc abc ================================================ FILE: tests/test-c/i_32_3.c ================================================ /* i_32_3.c: Character constant in #if expression. */ #include "defs.h" main( void) { fputs( "started\n", stderr); /* 32.3: */ #if 'a' != 0x61 fputs( "Not ASCII character set, or bad evaluation of character constant." , stderr); exit( 1); #endif /* 32.4: '\a' and '\v' */ #if '\a' != 7 || '\v' != 11 fputs( "Not ASCII character set, or bad evaluation of escape sequences." , stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/i_35.c ================================================ /* i_35.c: Multi-character character constant. */ #include "defs.h" main( void) { fputs( "started\n", stderr); /* In ASCII character set. */ /* 35.1: */ #if ('ab' != '\x61\x62') || ('\aa' != '\7\x61') fputs( "Bad handling of multi-character character constant.\n", stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/i_35_3.c ================================================ /* i_35_3.c: Multi-character wide character constant. */ #include "defs.h" main( void) { fputs( "started\n", stderr); /* In ASCII character set. */ /* 35.3: */ #if (L'ab' != L'\x61\x62') || (L'ab' == 'ab') fputs( "Bad handling of multi-character wide character constant.\n", stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/ifdef15.h ================================================ /* ifdef15.h */ #ifdef X01 #else #ifdef X02 #else #ifdef X03 #else #ifdef X04 #else #ifdef X05 #else #ifdef X06 #else #ifdef X07 #else #ifdef X08 #else #ifdef X09 #else #ifdef X0A #else #ifdef X0B #else #ifdef X0C #else #ifdef X0D #else #ifdef X0E #else #ifdef X0F nest = 0x0f; #endif /* X0F */ #endif /* X0E */ #endif /* X0D */ #endif /* X0C */ #endif /* X0B */ #endif /* X0A */ #endif /* X09 */ #endif /* X08 */ #endif /* X07 */ #endif /* X06 */ #endif /* X05 */ #endif /* X04 */ #endif /* X03 */ #endif /* X02 */ #endif /* X01 */ ================================================ FILE: tests/test-c/line.h ================================================ /* line.h */ { char * file = __FILE__; file += strlen( file) - 6; assert( __LINE__ == 6 && strcmp( file, "line.h") == 0); } ================================================ FILE: tests/test-c/m1024.h ================================================ /* m1024.h */ #define AA #define AB #define AC #define AD #define AE #define AF #define AG #define AH #define AI #define AJ #define AK #define AL #define AM #define AN #define AO #define AP #define AQ #define AR #define AS #define AT #define AU #define AV #define AW #define AX #define AY #define AZ #define Aa #define Ab #define Ac #define Ad #define Ae #define Af #define Ag #define Ah #define Ai #define Aj #define Ak #define Al #define Am #define An #define BA #define BB #define BC #define BD #define BE #define BF #define BG #define BH #define BI #define BJ #define BK #define BL #define BM #define BN #define BO #define BP #define BQ #define BR #define BS #define BT #define BU #define BV #define BW #define BX #define BY #define BZ #define Ba #define Bb #define Bc #define Bd #define Be #define Bf #define Bg #define Bh #define Bi #define Bj #define Bk #define Bl #define Bm #define Bn #define CA #define CB #define CC #define CD #define CE #define CF #define CG #define CH #define CI #define CJ #define CK #define CL #define CM #define CN #define CO #define CP #define CQ #define CR #define CS #define CT #define CU #define CV #define CW #define CX #define CY #define CZ #define Ca #define Cb #define Cc #define Cd #define Ce #define Cf #define Cg #define Ch #define Ci #define Cj #define Ck #define Cl #define Cm #define Cn #define DA #define DB #define DC #define DD #define DE #define DF #define DG #define DH #define DI #define DJ #define DK #define DL #define DM #define DN #define DO #define DP #define DQ #define DR #define DS #define DT #define DU #define DV #define DW #define DX #define DY #define DZ #define Da #define Db #define Dc #define Dd #define De #define Df #define Dg #define Dh #define Di #define Dj #define Dk #define Dl #define Dm #define Dn #define EA #define EB #define EC #define ED #define EE #define EF #define EG #define EH #define EI #define EJ #define EK #define EL #define EM #define EN #define EO #define EP #define EQ #define ER #define ES #define ET #define EU #define EV #define EW #define EX #define EY #define EZ #define Ea #define Eb #define Ec #define Ed #define Ee #define Ef #define Eg #define Eh #define Ei #define Ej #define Ek #define El #define Em #define En #define FA #define FB #define FC #define FD #define FE #define FF #define FG #define FH #define FI #define FJ #define FK #define FL #define FM #define FN #define FO #define FP #define FQ #define FR #define FS #define FT #define FU #define FV #define FW #define FX #define FY #define FZ #define Fa #define Fb #define Fc #define Fd #define Fe #define Ff #define Fg #define Fh #define Fi #define Fj #define Fk #define Fl #define Fm #define Fn #define GA #define GB #define GC #define GD #define GE #define GF #define GG #define GH #define GI #define GJ #define GK #define GL #define GM #define GN #define GO #define GP #define GQ #define GR #define GS #define GT #define GU #define GV #define GW #define GX #define GY #define GZ #define Ga #define Gb #define Gc #define Gd #define Ge #define Gf #define Gg #define Gh #define Gi #define Gj #define Gk #define Gl #define Gm #define Gn #define HA #define HB #define HC #define HD #define HE #define HF #define HG #define HH #define HI #define HJ #define HK #define HL #define HM #define HN #define HO #define HP #define HQ #define HR #define HS #define HT #define HU #define HV #define HW #define HX #define HY #define HZ #define Ha #define Hb #define Hc #define Hd #define He #define Hf #define Hg #define Hh #define Hi #define Hj #define Hk #define Hl #define Hm #define Hn #define IA #define IB #define IC #define ID #define IE #define IF #define IG #define IH #define II #define IJ #define IK #define IL #define IM #define IN #define IO #define IP #define IQ #define IR #define IS #define IT #define IU #define IV #define IW #define IX #define IY #define IZ #define Ia #define Ib #define Ic #define Id #define Ie #define If #define Ig #define Ih #define Ii #define Ij #define Ik #define Il #define Im #define In #define JA #define JB #define JC #define JD #define JE #define JF #define JG #define JH #define JI #define JJ #define JK #define JL #define JM #define JN #define JO #define JP #define JQ #define JR #define JS #define JT #define JU #define JV #define JW #define JX #define JY #define JZ #define Ja #define Jb #define Jc #define Jd #define Je #define Jf #define Jg #define Jh #define Ji #define Jj #define Jk #define Jl #define Jm #define Jn #define KA #define KB #define KC #define KD #define KE #define KF #define KG #define KH #define KI #define KJ #define KK #define KL #define KM #define KN #define KO #define KP #define KQ #define KR #define KS #define KT #define KU #define KV #define KW #define KX #define KY #define KZ #define Ka #define Kb #define Kc #define Kd #define Ke #define Kf #define Kg #define Kh #define Ki #define Kj #define Kk #define Kl #define Km #define Kn #define LA #define LB #define LC #define LD #define LE #define LF #define LG #define LH #define LI #define LJ #define LK #define LL #define LM #define LN #define LO #define LP #define LQ #define LR #define LS #define LT #define LU #define LV #define LW #define LX #define LY #define LZ #define La #define Lb #define Lc #define Ld #define Le #define Lf #define Lg #define Lh #define Li #define Lj #define Lk #define Ll #define Lm #define Ln #define MA #define MB #define MC #define MD #define ME #define MF #define MG #define MH #define MI #define MJ #define MK #define ML #define MM #define MN #define MO #define MP #define MQ #define MR #define MS #define MT #define MU #define MV #define MW #define MX #define MY #define MZ #define Ma #define Mb #define Mc #define Md #define Me #define Mf #define Mg #define Mh #define Mi #define Mj #define Mk #define Ml #define Mm #define Mn #define NA #define NB #define NC #define ND #define NE #define NF #define NG #define NH #define NI #define NJ #define NK #define NL #define NM #define NN #define NO #define NP #define NQ #define NR #define NS #define NT #define NU #define NV #define NW #define NX #define NY #define NZ #define Na #define Nb #define Nc #define Nd #define Ne #define Nf #define Ng #define Nh #define Ni #define Nj #define Nk #define Nl #define Nm #define Nn #define OA #define OB #define OC #define OD #define OE #define OF #define OG #define OH #define OI #define OJ #define OK #define OL #define OM #define ON #define OO #define OP #define OQ #define OR #define OS #define OT #define OU #define OV #define OW #define OX #define OY #define OZ #define Oa #define Ob #define Oc #define Od #define Oe #define Of #define Og #define Oh #define Oi #define Oj #define Ok #define Ol #define Om #define On #define PA #define PB #define PC #define PD #define PE #define PF #define PG #define PH #define PI #define PJ #define PK #define PL #define PM #define PN #define PO #define PP #define PQ #define PR #define PS #define PT #define PU #define PV #define PW #define PX #define PY #define PZ #define Pa #define Pb #define Pc #define Pd #define Pe #define Pf #define Pg #define Ph #define Pi #define Pj #define Pk #define Pl #define Pm #define Pn #define QA #define QB #define QC #define QD #define QE #define QF #define QG #define QH #define QI #define QJ #define QK #define QL #define QM #define QN #define QO #define QP #define QQ #define QR #define QS #define QT #define QU #define QV #define QW #define QX #define QY #define QZ #define Qa #define Qb #define Qc #define Qd #define Qe #define Qf #define Qg #define Qh #define Qi #define Qj #define Qk #define Ql #define Qm #define Qn #define RA #define RB #define RC #define RD #define RE #define RF #define RG #define RH #define RI #define RJ #define RK #define RL #define RM #define RN #define RO #define RP #define RQ #define RR #define RS #define RT #define RU #define RV #define RW #define RX #define RY #define RZ #define Ra #define Rb #define Rc #define Rd #define Re #define Rf #define Rg #define Rh #define Ri #define Rj #define Rk #define Rl #define Rm #define Rn #define SA #define SB #define SC #define SD #define SE #define SF #define SG #define SH #define SI #define SJ #define SK #define SL #define SM #define SN #define SO #define SP #define SQ #define SR #define SS #define ST #define SU #define SV #define SW #define SX #define SY #define SZ #define Sa #define Sb #define Sc #define Sd #define Se #define Sf #define Sg #define Sh #define Si #define Sj #define Sk #define Sl #define Sm #define Sn #define TA #define TB #define TC #define TD #define TE #define TF #define TG #define TH #define TI #define TJ #define TK #define TL #define TM #define TN #define TO #define TP #define TQ #define TR #define TS #define TT #define TU #define TV #define TW #define TX #define TY #define TZ #define Ta #define Tb #define Tc #define Td #define Te #define Tf #define Tg #define Th #define Ti #define Tj #define Tk #define Tl #define Tm #define Tn #define UA #define UB #define UC #define UD #define UE #define UF #define UG #define UH #define UI #define UJ #define UK #define UL #define UM #define UN #define UO #define UP #define UQ #define UR #define US #define UT #define UU #define UV #define UW #define UX #define UY #define UZ #define Ua #define Ub #define Uc #define Ud #define Ue #define Uf #define Ug #define Uh #define Ui #define Uj #define Uk #define Ul #define Um #define Un #define VA #define VB #define VC #define VD #define VE #define VF #define VG #define VH #define VI #define VJ #define VK #define VL #define VM #define VN #define VO #define VP #define VQ #define VR #define VS #define VT #define VU #define VV #define VW #define VX #define VY #define VZ #define Va #define Vb #define Vc #define Vd #define Ve #define Vf #define Vg #define Vh #define Vi #define Vj #define Vk #define Vl #define Vm #define Vn #define WA #define WB #define WC #define WD #define WE #define WF #define WG #define WH #define WI #define WJ #define WK #define WL #define WM #define WN #define WO #define WP #define WQ #define WR #define WS #define WT #define WU #define WV #define WW #define WX #define WY #define WZ #define Wa #define Wb #define Wc #define Wd #define We #define Wf #define Wg #define Wh #define Wi #define Wj #define Wk #define Wl #define Wm #define Wn #define XA #define XB #define XC #define XD #define XE #define XF #define XG #define XH #define XI #define XJ #define XK #define XL #define XM #define XN #define XO #define XP #define XQ #define XR #define XS #define XT #define XU #define XV #define XW #define XX #define XY #define XZ #define Xa #define Xb #define Xc #define Xd #define Xe #define Xf #define Xg #define Xh #define Xi #define Xj #define Xk #define Xl #define Xm #define Xn #define YA #define YB #define YC #define YD #define YE #define YF #define YG #define YH #define YI #define YJ #define YK #define YL #define YM #define YN #define YO #define YP #define YQ #define YR #define YS #define YT #define YU #define YV #define YW #define YX #define YY #define YZ #define Ya #define Yb #define Yc #define Yd #define Ye #define Yf #define Yg #define Yh #define Yi #define Yj #define Yk #define Yl #define Ym #define Yn #define ZA #define ZB #define ZC #define ZD #define ZE #define ZF #define ZG #define ZH #define ZI #define ZJ #define ZK #define ZL #define ZM #define ZN #define ZO #define ZP #define ZQ #define ZR #define ZS #define ZT #define ZU #define ZV #define ZW #define ZX 1 ================================================ FILE: tests/test-c/m_33_big5.c ================================================ /* m_33_big5.c: Wide character constant encoded in Big-Five. */ #include "defs.h" #include #define BYTES_VAL (1 << CHAR_BIT) main( void) { char * ptr; fputs( "started\n", stderr); /* 33.1: L'ch'. */ #pragma __setlocale( "big5") /* For MCPP */ #pragma setlocale( "chinese-traditional") /* For Visual C */ #if L'r' == '\xa6' * BYTES_VAL + '\x72' ptr = "Wide character is encoded in Big-Five."; #elif L'r' == '\x72' * BYTES_VAL + '\xa6' ptr = "Wide character is encoded in Big-Five." "Inverted order of evaluation."; #else ptr = "I cannot understand Big-Five."; #endif assert( strcmp( ptr, "I cannot understand Big-Five.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_33_eucjp.c ================================================ /* m_33_eucjp.c: Wide character constant encoded in EUC-JP. */ #include "defs.h" #include #define BYTES_VAL (1 << CHAR_BIT) main( void) { char * ptr; fputs( "started\n", stderr); /* 33.1: L'ch'. */ #pragma __setlocale( "eucjp") /* For MCPP */ #pragma setlocale( "eucjp") /* For MCPP on VC */ #if L'' == '\xbb' * BYTES_VAL + '\xfa' ptr = "Wide character is encoded in EUC-JP."; #elif L'' == '\xfa' * BYTES_VAL + '\xbb' ptr = "Wide character is encoded in EUC-JP." "Inverted order of evaluation."; #else ptr = "I cannot understand EUC-JP."; #endif assert( strcmp( ptr, "I cannot understand EUC-JP.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_33_gb.c ================================================ /* m_33_gb.c: Wide character constant encoded in GB-2312. */ #include "defs.h" #include #define BYTES_VAL (1 << CHAR_BIT) main( void) { char * ptr; fputs( "started\n", stderr); /* 33.1: L'ch'. */ #pragma __setlocale( "gb2312") /* For MCPP */ #pragma setlocale( "chinese-simplified") /* For Visual C */ #if L'' == '\xd7' * BYTES_VAL + '\xd6' ptr = "Wide character is encoded in GB 2312."; #elif L'' == '\xd6' * BYTES_VAL + '\xd7' ptr = "Wide character is encoded in GB 2312." "Inverted order of evaluation."; #else ptr = "I cannot understand GB-2312."; #endif assert( strcmp( ptr, "I cannot understand GB-2312.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_33_jis.c ================================================ /* m_33_jis.c: Wide character constant encoded in ISO-2022-JP. */ #include "defs.h" #include #define BYTES_VAL (1 << CHAR_BIT) main( void) { char * ptr; fputs( "started\n", stderr); /* 33.1: L'ch'. */ #pragma __setlocale( "jis") /* For MCPP */ #pragma setlocale( "jis") /* For MCPP on VC */ #if L'$B;z(B' == 0x3b * BYTES_VAL + 0x7a /* This line doesn't work unless "shift states" are processed. */ ptr = "Wide character is encoded in ISO-2022-JP."; #elif L'$B;z(B' == 0x7a * BYTES_VAL + 0x3b /* This line doesn't work unless "shift states" are processed. */ ptr = "Wide character is encoded in ISO-2022-JP." "Inverted order of evaluation."; #else ptr = "I cannot understand ISO-2022-JP."; #endif assert( strcmp( ptr, "I cannot understand ISO-2022-JP.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_33_ksc.c ================================================ /* m_33_ksc.c: Wide character constant encoded in KSC-5601. */ #include "defs.h" #include #define BYTES_VAL (1 << CHAR_BIT) main( void) { char * ptr; fputs( "started\n", stderr); /* 33.1: L'ch'. */ #pragma __setlocale( "ksc5601") /* For MCPP */ #pragma setlocale( "korean") /* For Visual C */ #if L'' == '\xed' * BYTES_VAL + '\xae' ptr = "Wide character is encoded in KSC-5601."; #elif L'' == '\xae' * BYTES_VAL + '\xed' ptr = "Wide character is encoded in KSC-5601." "Inverted order of evaluation."; #else ptr = "I cannot understand KSC-5601."; #endif assert( strcmp( ptr, "I cannot understand KSC-5601.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_33_sjis.c ================================================ /* m_33_sjis.c: Wide character constant encoded in shift-JIS. */ #include "defs.h" #include #define BYTES_VAL (1 << CHAR_BIT) main( void) { char * ptr; fputs( "started\n", stderr); /* 33.1: L'ch'. */ #pragma __setlocale( "sjis") /* For MCPP */ #pragma setlocale( "japanese") /* For Visual C */ #if L'' == '\x8e' * BYTES_VAL + '\x9a' ptr = "Wide character is encoded in shift-JIS."; #elif L'' == '\x9a' * BYTES_VAL + '\x8e' ptr = "Wide character is encoded in shift-JIS." "Inverted order of evaluation."; #else ptr = "I cannot understand shift-JIS."; #endif assert( strcmp( ptr, "I cannot understand shift-JIS.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_33_utf8.c ================================================ /* m_33_utf8.c: Wide character constant encoded in UTF-8. */ #include "defs.h" #include #define BYTES_VAL (1 << CHAR_BIT) main( void) { char * ptr; fputs( "started\n", stderr); /* 33.1: L'ch'. */ #pragma __setlocale( "utf8") /* For MCPP */ #pragma setlocale( "utf8") /* For MCPP on VC */ #if L'字' == '\xe5' * BYTES_VAL * BYTES_VAL + '\xad' * BYTES_VAL + '\x97' ptr = "Wide character is encoded in UTF-8."; #elif L'字' == '\x97' * BYTES_VAL * BYTES_VAL + '\xad' * BYTES_VAL + '\xe5' ptr = "Wide character is encoded in UTF-8." "Inverted order of evaluation."; #else ptr = "I cannot understand UTF-8."; #endif assert( strcmp( ptr, "I cannot understand UTF-8.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_34_big5.c ================================================ /* m_34_big5.t: Multi-byte character constant encoded in Big-Five. */ #include "defs.h" main( void) { char * ptr; fputs( "started\n", stderr); /* 34.1: */ #pragma __setlocale( "big5") /* For MCPP */ #pragma setlocale( "chinese-traditional") /* For Visual C */ #if 'r' == '\xa6\x72' ptr = "Multi-byte character is encoded in Big-Five."; #else ptr = "I cannot understand Big-Five."; #endif assert( strcmp( ptr, "I cannot understand Big-Five.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_34_eucjp.c ================================================ /* m_34_eucjp.c: Multi-byte character constant encoded in EUC-JP. */ #include "defs.h" main( void) { char * ptr; fputs( "started\n", stderr); /* 34.1: */ #pragma __setlocale( "eucjp") /* For MCPP */ #pragma setlocale( "eucjp") /* For MCPP on VC */ #if '' == '\xbb\xfa' ptr = "Multi-byte character is encoded in EUC-JP."; #else ptr = "I cannot understand EUC-JP."; #endif assert( strcmp( ptr, "I cannot understand EUC-JP.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_34_gb.c ================================================ /* m_34_gb.c: Multi-byte character constant encoded in GB-2312. */ #include "defs.h" main( void) { char * ptr; fputs( "started\n", stderr); /* 34.1: */ #pragma __setlocale( "gb2312") /* For MCPP */ #pragma setlocale( "chinese-simplified") /* For Visual C */ #if '' == '\xd7\xd6' ptr = "Multi-byte character is encoded in GB-2312."; #else ptr = "I cannot understand GB-2312."; #endif assert( strcmp( ptr, "I cannot understand GB-2312.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_34_jis.c ================================================ /* m_34_jis.c: Multi-byte character constant encoded in ISO-2022-JP. */ #include "defs.h" main( void) { char * ptr; fputs( "started\n", stderr); /* 34.1: */ #pragma __setlocale( "jis") /* For MCPP */ #pragma setlocale( "jis") /* For MCPP on VC */ #if '$B;z(B' == '\x3b\x7a' /* This line doesn't work unless "shift states" are processed. */ ptr = "Multi-byte character is encoded in ISO-2022-JP."; #else ptr = "I cannot understand ISO-2022-JP."; #endif assert( strcmp( ptr, "I cannot understand ISO-2022-JP.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_34_ksc.c ================================================ /* m_34_ksc.c: Multi-byte character constant encoded in KSC-5601. */ #include "defs.h" main( void) { char * ptr; fputs( "started\n", stderr); /* 34.1: */ #pragma __setlocale( "ksc5601") /* For MCPP */ #pragma setlocale( "korean") /* For Visual C */ #if '' == '\xed\xae' ptr = "Multi-byte character is encoded in KSC-5601."; #else ptr = "I cannot understand KSC-5601."; #endif assert( strcmp( ptr, "I cannot understand KSC-5601.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_34_sjis.c ================================================ /* m_34_sjis.c: Multi-byte character constant encoded in shift-JIS. */ #include "defs.h" main( void) { char * ptr; fputs( "started\n", stderr); /* 34.1: */ #pragma __setlocale( "sjis") /* For MCPP */ #pragma setlocale( "japanese") /* For Visual C */ #if '' == '\x8e\x9a' ptr = "Multi-byte character is encoded in shift-JIS."; #else ptr = "I cannot understand shift-JIS."; #endif assert( strcmp( ptr, "I cannot understand shift-JIS.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_34_utf8.c ================================================ /* m_34_utf8.c: Multi-byte character constant encoded in UTF-8. */ #include "defs.h" main( void) { char * ptr; fputs( "started\n", stderr); /* 34.1: */ #pragma __setlocale( "utf8") /* For MCPP */ #pragma setlocale( "utf8") /* For MCPP on VC */ #if '字' == '\xe5\xad\x97' ptr = "Multi-byte character is encoded in UTF-8."; #else ptr = "I cannot understand UTF-8."; #endif assert( strcmp( ptr, "I cannot understand UTF-8.") != 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/m_36_big5.c ================================================ /* m_36_big5.c: Handling of '\\' in BigFive multi-byte character. */ #include "defs.h" #define str( a) # a main( void) { fputs( "started\n", stdout); /* 36.1: 0x5c in multi-byte character is not an escape character */ #pragma __setlocale( "big5") /* For MCPP */ #pragma setlocale( "chinese-traditional") /* For Visual C */ #if 'r' == '\xa6\x72' && '\' != '\xa5\x5c' fputs( "Bad handling of '\\' in multi-byte character", stdout); exit( 1); #endif /* 36.2: # operater should not insert '\\' before 0x5c in multi-byte character */ assert( strcmp( str( "\Z"), "\"\Z\"") == 0); fputs( "\Z" "\"\Z\"\n", stdout); fputs( "success\n", stdout); return 0; } ================================================ FILE: tests/test-c/m_36_jis.c ================================================ /* m_36_jis.c: Handling of '\\' in ISO-2022-JP multi--byte character. */ #include "defs.h" #define str( a) # a main( void) { fputs( "started\n", stdout); /* 36.1: 0x5c in multi-byte character is not an escape character */ #pragma __setlocale( "jis") /* For MCPP */ #pragma setlocale( "jis") /* For MCPP on VC */ #if '$B;z(B' == '\x3b\x7a' && '$B0\(B' != '\x30\x5c' fputs( "Bad handling of '\\' in multi-byte character", stdout); exit( 1); #endif /* 36.2: # operater should not insert '\\' before 0x5c, 0x22 or 0x27 in multi-byte character */ assert( strcmp( str( "$B0\F0(B"), "\"$B0\F0(B\"") == 0); assert( strcmp( str( "$B1"M[(B"), "\"$B1"M[(B\"") == 0); assert( strcmp( str( "$B1'Ch(B"), "\"$B1'Ch(B\"") == 0); fputs( "$B0\F0(B" "\"$B0\F0(B\"\n", stdout); fputs( "$B1"M[(B" "\"$B1"M[(B\"\n", stdout); fputs( "$B1'Ch(B" "\"$B1'Ch(B\"\n", stdout); fputs( "success\n", stdout); return 0; } ================================================ FILE: tests/test-c/m_36_sjis.c ================================================ /* m_36_sjis.c: Handling of '\\' in shift-JIS multi-byte character. */ #include "defs.h" #define str( a) # a main( void) { fputs( "started\n", stdout); /* 36.1: 0x5c in multi-byte character is not an escape character */ #pragma __setlocale( "sjis") /* For MCPP */ #pragma setlocale( "japanese") /* For Visual C */ #if '' == '\x8e\x9a' && '\' != '\x95\x5c' fputs( "Bad handling of '\\' in multi-byte character", stdout); exit( 1); #endif /* 36.2: # operater should not insert '\\' before 0x5c in multi-byte character */ assert( strcmp( str( "\"), "\"\\"") == 0); fputs( "\" "\"\\"\n", stdout); fputs( "success\n", stdout); return 0; } ================================================ FILE: tests/test-c/n_1.c ================================================ /* n_1.c: Conversion of trigraph sequences. */ #include "defs.h" char quasi_trigraph[] = { '?', '?', ' ', '?', '?', '?', ' ', '?', '?', '%', ' ', '?', '?', '^', ' ', '?', '#', '\0' }; main( void) { int ab = 1, cd = 2; fputs( "started\n", stderr); /* 1.1: The following 9 sequences are valid trigraph sequences. */ assert( strcmp( "??( ??) ??/??/ ??' ??< ??> ??! ??- ??=" ,"[ ] \\ ^ { } | ~ #") == 0); /* 1.2: In directive line. */ ??= define OR( a, b) a ??! b assert( OR( ab, cd) == 3); /* 1.3: Any sequence other than above 9 is not a trigraph sequence. */ assert( strcmp( "?? ??? ??% ??^ ???=", quasi_trigraph) == 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_10.c ================================================ /* n_10.c: #if, #elif, #else and #endif pp-directive. */ #include "defs.h" #define MACRO_0 0 #define MACRO_1 1 main( void) { fputs( "started\n", stderr); /* 10.1: */ /* Note: an undefined identifier in #if expression is replaced to 0. */ #if a assert( a); #elif MACRO_0 assert( MACRO_0); #elif MACRO_1 /* Valid block */ assert( MACRO_1); #else assert( 0); #endif /* 10.2: Comments must be processed even if in skipped #if block. */ /* At least tokenization of string literal and character constant is necessary to process comments, e.g. /* is not a comment mark in string literal. */ #ifdef UNDEFINED /* Comment */ "in literal /* is not a comment" #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_11.c ================================================ /* n_11.c: Operator "defined" in #if or #elif directive. */ #include "defs.h" #define MACRO_abc abc #define MACRO_0 0 #define ZERO_TOKEN main( void) { int abc = 1, a = 0; fputs( "started\n", stderr); /* 11.1: */ #if defined a assert( a); #else assert( MACRO_abc); #endif #if defined (MACRO_abc) assert( MACRO_abc); #else assert( a); #endif /* 11.2: "defined" is an unary operator whose result is 1 or 0. */ #if defined MACRO_0 * 3 != 3 fputs( "Bad handling of defined operator.\n", stderr); exit( 1); #endif #if (!defined ZERO_TOKEN != 0) || (-defined ZERO_TOKEN != -1) fputs( "Bad grouping of defined, -, ! in #if expression.\n", stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_12.c ================================================ /* n_12.c: Integer preprocessing number token and type of #if expression. */ #include "defs.h" #include main( void) { fputs( "started\n", stderr); /* 12.1: */ #if LONG_MAX <= LONG_MIN fputs( "Bad evaluation of long.\n", stderr); exit( 1); #endif #if LONG_MAX <= 1073741823 /* 0x3FFFFFFF */ fputs( "Bad evaluation of long.\n", stderr); exit( 1); #endif /* 12.2: */ #if ULONG_MAX / 2 < LONG_MAX fputs( "Bad evaluation of unsigned long.\n", stderr); exit( 1); #endif /* 12.3: Octal number. */ #if 0177777 != 65535 fputs( "Bad evaluation of octal number.\n", stderr); exit( 1); #endif /* 12.4: Hexadecimal number. */ #if 0Xffff != 65535 || 0xFfFf != 65535 fputs( "Bad evaluation of hexadecimal number.\n", stderr); exit( 1); #endif /* 12.5: Suffix 'L' or 'l'. */ #if 0L != 0 || 0l != 0 fputs( "Bad evaluation of 'L' suffix.\n", stderr); exit( 1); #endif /* 12.6: Suffix 'U' or 'u'. */ #if 1U != 1 || 1u != 1 fputs( "Bad evaluation of 'U' suffix.\n", stderr); exit( 1); #endif /* 12.7: Negative integer. */ #if 0 <= -1 fputs( "Bad evaluation of negative number.\n", stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_13.c ================================================ /* n_13.c: Valid operators in #if expression. */ /* Valid operators are (precedence in this order) : defined, (unary)+, (unary)-, ~, !, *, /, %, +, -, <<, >>, <, >, <=, >=, ==, !=, &, ^, |, &&, ||, ? : */ #include "defs.h" main( void) { fputs( "started\n", stderr); /* 13.1: Bit shift. */ #if 1 << 2 != 4 || 8 >> 1 != 4 fputs( "Bad arithmetic of <<, >> operators.\n", stderr); exit( 1); #endif /* 13.2: Bitwise operators. */ #if (3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1 fputs( "Bad arithmetic of ^, |, & operators.\n", stderr); exit( 1); #endif /* 13.3: Result of ||, && operators is either of 1 or 0. */ #if (2 || 3) != 1 || (2 && 3) != 1 || (0 || 4) != 1 || (0 && 5) != 0 fputs( "Bad arithmetic of ||, && operators.\n", stderr); exit( 1); #endif /* 13.4: ?, : operator. */ #if (0 ? 1 : 2) != 2 fputs( "Bad arithmetic of ?: operator.\n", stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_13_13.c ================================================ /* n_13_13.c: #if expression with macros. */ #include "defs.h" #define ZERO_TOKEN #define MACRO_0 0 #define MACRO_1 1 #define and && #define or || #define not_eq != #define bitor | main( void) { fputs( "started\n", stderr); /* 13.13: With macros expanding to operators. */ #if (1 bitor 2) == 3 and 4 not_eq 5 or 0 /* #if (1 | 2) == 3 && 4 != 5 || 0 */ #else fputs( "Bad evaluation of macros expanding to operators in #if expression.\n" , stderr); exit( 1); #endif /* 13.14: With macros expanding to 0 token, nonsence but legal. */ #if ZERO_TOKEN MACRO_1 ZERO_TOKEN > ZERO_TOKEN MACRO_0 ZERO_TOKEN /* #if 1 > 0 */ #else fputs( "Bad evaluation of macros expanding to 0 token in #if expression.\n" , stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_13_5.c ================================================ /* n_13_5.c: Arithmetic conversion in #if expressions. */ #include "defs.h" main( void) { fputs( "started\n", stderr); /* 13.5: The usual arithmetic conversion is not performed on bit shift. */ #if -1 << 3U > 0 fputs( "Bad conversion of bit shift operands.\n", stderr); exit( 1); #endif /* 13.6: Usual arithmetic conversions. */ #if -1 <= 0U /* -1 is converted to unsigned long. */ fputs( "Bad arithmetic conversion.\n", stderr); exit( 1); #endif #if -1 * 1U <= 0 fputs( "Bad arithmetic conversion.\n", stderr); exit( 1); #endif /* Second and third operands of conditional operator are converted to the same type, thus -1 is converted to unsigned long. */ #if (1 ? -1 : 0U) <= 0 fputs( "Bad arithmetic conversion.\n", stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_13_7.c ================================================ /* n_13_7.c: Short-circuit evaluation of #if expression. */ #include "defs.h" #define MACRO_0 0 main( void) { fputs( "started\n", stderr); /* 13.7: 10/0 or 10/MACRO_0 are never evaluated, "divide by zero" error cannot occur. */ #if 0 && 10 / 0 exit( 1); #endif #if not_defined && 10 / not_defined exit( 1); #endif #if MACRO_0 && 10 / MACRO_0 > 1 exit( 1); #endif #if MACRO_0 ? 10 / MACRO_0 : 0 exit( 1); #endif #if MACRO_0 == 0 || 10 / MACRO_0 > 1 /* Valid block */ fputs( "success\n", stderr); return 0; #else exit( 1); #endif } ================================================ FILE: tests/test-c/n_13_8.c ================================================ /* n_13_8.c: Grouping of sub-expressions in #if expression. */ #include "defs.h" main( void) { fputs( "started\n", stderr); /* 13.8: Unary operators are grouped from right to left. */ #if (- -1 != 1) || (!!9 != 1) || (-!+!9 != -1) || (~~1 != 1) fputs( "Bad grouping of -, +, !, ~ in #if expression.\n", stderr); exit( 1); #endif /* 13.9: ?: operators are grouped from right to left. */ #if (1 ? 2 ? 3 ? 3 : 2 : 1 : 0) != 3 fputs( "Bad grouping of ? : in #if expression.\n", stderr); exit( 1); #endif /* 13.10: Other operators are grouped from left to right. */ #if (15 >> 2 >> 1 != 1) || (3 << 2 << 1 != 24) fputs( "Bad grouping of >>, << in #if expression.\n", stderr); exit( 1); #endif /* 13.11: Test of precedence. */ #if 3*10/2 >> !0*2 >> !+!-9 != 1 fputs( "Bad grouping of -, +, !, *, /, >> in #if expression.\n", stderr); exit( 1); #endif /* 13.12: Overall test. Grouped as: ((((((+1 - -1 - ~~1 - -!0) & 6) | ((8 % 9) ^ (-2 * -2))) >> 1) == 7) ? 7 : 0) != 7 evaluated to FALSE. */ #if (((+1- -1-~~1- -!0&6|8%9^-2*-2)>>1)==7?7:0)!=7 fputs( "Bad arithmetic of #if expression.\n", stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_15.c ================================================ /* n_15.c: #ifdef, #ifndef directives. */ #include "defs.h" #define MACRO_0 0 #define MACRO_1 1 main( void) { fputs( "started\n", stderr); /* 15.1: #ifdef directive. */ #ifdef MACRO_1 /* Valid block */ assert( MACRO_1); #else assert( MACRO_0); #endif /* 15.2: #ifndef directive. */ #ifndef MACRO_1 assert( MACRO_0); #else /* Valid block */ assert( MACRO_1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_18.c ================================================ /* n_18.c: #define directive. */ #include "defs.h" main( void) { int c = 3; /* Excerpts from ISO C 6.8.3 "Examples". */ #define OBJ_LIKE (1-1) #define FTN_LIKE(a) ( a ) fputs( "started\n", stderr); /* 18.1: Definition of an object-like macro. */ assert( OBJ_LIKE == 0); #define ZERO_TOKEN #ifndef ZERO_TOKEN fputs( "Can't define macro to 0-token.\n", stderr); exit( 1); #endif /* 18.2: Definition of a function-like macro. */ assert( FTN_LIKE( c) == 3); /* 18.3: Spelling in string identical to parameter is not a parameter. */ #define STR( n1, n2) "n1:n2" assert( strcmp( STR( 1, 2), "n1:n2") == 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_19.c ================================================ /* n_19.c: Valid re-definitions of macros. */ #include "defs.h" main( void) { int c = 1; fputs( "started\n", stderr); /* Excerpts from ISO C 6.8.3 "Examples". */ #define OBJ_LIKE (1-1) #define FTN_LIKE(a) ( a ) /* 19.1: */ #define OBJ_LIKE /* white space */ (1-1) /* other */ /* 19.2: */ #define FTN_LIKE( a )( /* note the white space */ \ a /* other stuff on this line */ ) assert( FTN_LIKE( c) == 1); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_2.c ================================================ /* n_2.c: Line splicing by sequence. */ #include "defs.h" main( void) { int ab = 1, cd = 2, ef = 3, abcde = 5; fputs( "started\n", stderr); /* 2.1: In a #define directive line, between the parameter list and the replacement text. */ #define FUNC( a, b, c) \ a + b + c assert( FUNC( ab, cd, ef) == 6); /* 2.2: In a #define directive line, among the parameter list and among the replacement text. */ #undef FUNC #define FUNC( a, b \ , c) \ a + b \ + c assert (FUNC( ab, cd, ef) == 6); /* 2.3: In a string literal. */ assert (strcmp( "abc\ de", "abcde") == 0); /* 2.4: in midst of an identifier. */ assert( abc\ de == 5); /* 2.5: by trigraph. */ assert( abc??/ de == 5); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_20.c ================================================ /* n_20.c: Macro lexically identical to keyword. */ #include "defs.h" main( void) { /* 20.1: */ #define float double float fl; fputs( "started\n", stderr); assert( sizeof fl == sizeof (double)); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_21.c ================================================ /* n_21.c: Tokenization (No preprocessing tokens are merged implicitly). */ #include "defs.h" main( void) { int a = 1, x = 2, y = -3; fputs( "started\n", stderr); /* 21.1: */ #define MINUS - assert( -MINUS-a == -1); /* 21.2: */ #define sub( a, b) a-b /* '(a)-(b)' is better */ #define Y -y /* '(-y)' is better */ /* x- -y */ assert( sub( x, Y) == -1); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_22.c ================================================ /* n_22.c: Tokenization of preprocessing number. */ #include "defs.h" #define str( a) # a #define xstr( a) str( a) #define EXP 1 main( void) { fputs( "started\n", stderr); /* 22.1: 12E+EXP is a preprocessing number, EXP is not expanded. */ assert( strcmp( xstr( 12E+EXP), "12E+EXP") == 0); /* 22.2: .2e-EXP is also a preprocessing number. */ assert( strcmp( xstr( .2e-EXP), ".2e-EXP") == 0); /* 22.3: + or - is allowed only following E or e, 12+EXP is not a preprocessing number. */ assert( strcmp( xstr( 12+EXP), "12+1") == 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_23.c ================================================ /* n_23.c: ## operator in macro definition. */ #include "defs.h" #define glue( a, b) a ## b #define xglue( a, b) glue( a, b) #define MACRO_0 0 #define MACRO_1 1 main( void) { int xy = 1; fputs( "started\n", stderr); /* 23.1: */ assert( glue( x, y) == 1); /* 23.2: Generate a preprocessing number. */ #define EXP 2 assert( xglue( .12e+, EXP) == 12.0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_24.c ================================================ /* n_24.c: # operator in macro definition. */ #include "defs.h" main( void) { fputs( "started\n", stderr); /* 24.1: */ #define str( a) # a assert( strcmp( str( a+b), "a+b") == 0); /* 24.2: White spaces between tokens of operand are converted to one space. */ assert( strcmp( str( ab /* comment */ + cd ), "ab + cd") == 0); /* 24.3: \ is inserted before \ and " in or surrounding literals and no other character is inserted to anywhere. */ assert( strcmp( str( '"' + "' \""), "'\"' + \"' \\\"\"") == 0); /* 24.4: Line splicing by is done prior to token parsing. */ assert( strcmp( str( "ab\ c"), "\"abc\"") == 0); /* 24.5: Token separator inserted by macro expansion should be removed. (Meanwhile, tokens should not be merged. See 21.2.) */ #define xstr( a) str( a) #define f(a) a assert( strcmp( xstr( x-f(y)), "x-y") == 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_25.c ================================================ /* n_25.c: Macro arguments are pre-expanded (unless the argument is an operand of # or ## operator) separately, that is, are macro-replaced completely prior to rescanning. */ #include "defs.h" #define ZERO_TOKEN #define MACRO_0 0 #define MACRO_1 1 #define TWO_ARGS a,b #define sub( x, y) (x - y) #define glue( a, b) a ## b #define xglue( a, b) glue( a, b) #define str( a) # a main( void) { int a = 1, b = 2, abc = 3, MACRO_0MACRO_1 = 2; fputs( "started\n", stderr); /* 25.1: "TWO_ARGS" is read as one argument to "sub", then expanded to "a,b", then "x" is substituted by "a,b". */ assert( sub( TWO_ARGS, 1) == 1); /* 25.2: An argument pre-expanded to 0-token. */ /* ( - 1); */ assert( sub( ZERO_TOKEN, a) == -1); /* 25.3: "glue( a, b)" is pre-expanded. */ assert( xglue( glue( a, b), c) == 3); /* 25.4: Operands of ## operator are not pre-expanded. */ assert( glue( MACRO_0, MACRO_1) == 2); /* 25.5: Operand of # operator is not pre-expanded. */ assert( strcmp( str( ZERO_TOKEN), "ZERO_TOKEN") == 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_26.c ================================================ /* n_26.c: The name once replaced is not furthur replaced. */ #include "defs.h" int f( a) int a; { return a; } int g( a) int a; { return a * 2; } main( void) { int x = 1; int AB = 1; int Z[1]; fputs( "started\n", stderr); Z[0] = 1; /* 26.1: Directly recursive object-like macro definition. */ /* Z[0]; */ #define Z Z[0] assert( Z == 1); /* 26.2: Intermediately recursive object-like macro definition. */ /* AB; */ #define AB BA #define BA AB assert( AB == 1); /* 26.3: Directly recursive function-like macro definition. */ /* x + f(x); */ #define f(a) a + f(a) assert( f( x) == 2); /* 26.4: Intermediately recursive function-like macro definition. */ /* x + x + g( x); */ #define g(a) a + h( a) #define h(a) a + g( a) assert( g( x) == 4); /* 26.5: Rescanning encounters the non-replaced macro name. */ /* Z[0] + f( Z[0]); */ assert( f( Z) == 2); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_27.c ================================================ /* n_27.c: Rescanning of a macro replace any macro call in the replacement text after substitution of parameters by pre-expanded-arguments. This re-examination may involve the succeding sequences from the source file (what a queer thing!). */ #include "defs.h" main( void) { int a = 1, b = 2, c, m = 1, n = 2; fputs( "started\n", stderr); /* 27.1: Cascaded use of object-like macros. */ /* 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8; */ #define NEST8 NEST7 + 8 #define NEST7 NEST6 + 7 #define NEST6 NEST5 + 6 #define NEST5 NEST4 + 5 #define NEST4 NEST3 + 4 #define NEST3 NEST2 + 3 #define NEST2 NEST1 + 2 #define NEST1 1 assert( NEST8 == 36); /* 27.2: Cascaded use of function-like macros. */ /* (1) + (1 + 2) + 1 + 2 + 1 + 2 + 3 + 1 + 2 + 3 + 4; */ #define FUNC4( a, b) FUNC3( a, b) + NEST4 #define FUNC3( a, b) FUNC2( a, b) + NEST3 #define FUNC2( a, b) FUNC1( a, b) + NEST2 #define FUNC1( a, b) (a) + (b) assert( FUNC4( NEST1, NEST2) == 23); /* 27.3: An identifier generated by ## operator is subject to expansion. */ #define glue( a, b) a ## b #define MACRO_1 1 assert( glue( MACRO_, 1) == 1); #define sub( x, y) (x - y) #define head sub( #define math( op, a, b) op( (a), (b)) /* 27.4: 'sub' as an argument of math() is not pre-expanded, since '(' is missing. */ assert( math( sub, a, b) == -1); /* 27.5: Queer thing. */ c = head a,b ); assert( c == -1); /* 27.6: Recursive macro (the 2nd 'm' is expanded to 'n' since it is in source file). */ #define m n #define n( a) a c = m( m); assert( c == 2); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_28.c ================================================ /* n_28.c: __FILE__, __LINE__, __DATE__, __TIME__, __STDC__ and __STDC_VERSION__ are predefined. */ #include "defs.h" main( void) { char * date = __DATE__; char * fname = __FILE__; fputs( "started\n", stderr); /* 28.1: */ /* Remove directory part (if any). */ fname += strlen( fname) - 6; assert( strcmp( fname, "n_28.c") == 0); /* 28.2: */ assert( __LINE__ == 19); /* 28.3: */ assert( strlen( __DATE__) == 11); assert( date[ 4] != '0'); /* 28.4: */ assert( strlen( __TIME__) == 8); /* 28.5: */ assert( __STDC__); /* 28.6: */ assert( __STDC_VERSION__); /* 28.7: */ #include "line.h" fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_29.c ================================================ /* n_29.c: #undef directive. */ #include "defs.h" main( void) { int DEFINED = 1; fputs( "started\n", stderr); /* 29.1: Undefined macro is not a macro. */ #define DEFINED #undef DEFINED assert( DEFINED == 1); /* 29.2: Undefining undefined name is not an error. */ #undef UNDEFINED fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_3.c ================================================ /* n_3.c: Handling of comment. */ #include "defs.h" #define str( a) # a main( void) { int abcd = 4; fputs( "started\n", stderr); /* 3.1: A comment is converted to one space. */ assert( strcmp( str( abc/* comment */de), "abc de") == 0); /* 3.2: // is not a comment of C. */ /* assert( strcmp( str( //), "//") == 0); */ /* 3.3: Comment is parsed prior to the parsing of preprocessing directive. */ #if 0 "nonsence"; /* #else still in comment */ #else #define MACRO_abcd /* in comment */ abcd #endif assert( MACRO_abcd == 4); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_30.c ================================================ /* n_30.c: Macro calls. */ /* Note: Comma separate the arguments of function-like macro call, but comma between matching inner parenthesis doesn't. This feature is tested on so many places in this suite especially on *.c samples which use assert() macro, that no separete item to test this feature is provided. */ #include "defs.h" #define FUNC( a, b, c) a + b + c main( void) { int a = 1, b = 2, c = 3; fputs( "started\n", stderr); /* 30.1: A macro call crossing lines. */ assert ( FUNC ( a, b, c ) == 6 ); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_32.c ================================================ /* n_32.c: Escape sequence in character constant in #if expression. */ #include "defs.h" main( void) { fputs( "started\n", stderr); /* 32.1: Character octal escape sequence. */ #if '\123' != 83 fputs( "Bad evaluation of octal escape sequence.\n", stderr); exit( 1); #endif /* 32.2: Character hexadecimal escape sequence. */ #if '\x1b' != '\033' fputs( "Bad evaluation of hexadecimal escape sequence.\n", stderr); exit( 1); #endif fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_37.c ================================================ /* n_37.c: Translation limits. */ #include "defs.h" /* 37.1: Number of parameters in macro: at least 31. */ #define glue31(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z,A,B,C,D,E) \ a##b##c##d##e##f##g##h##i##j##k##l##m##n##o##p##q##r##s##t##u##v##w##x##y##z##A##B##C##D##E main( void) { int ABCDEFGHIJKLMNOPQRSTUVWXYZabcde = 31; int ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ = 30; int nest = 0; fputs( "started\n", stderr); /* 37.2: Number of arguments in macro call: at least 31. */ assert( glue31( A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R , S, T, U, V, W, X, Y, Z, a, b, c, d, e) == 31); /* 37.3: Significant initial characters in an internal identifier or a macro name: at least 31. */ assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ == 30); /* 37.4: Nested conditional inclusion: at least 8 levels. */ nest = 0; #ifdef A #else # ifdef B # else # ifdef C # else # ifdef D # else # ifdef E # else # ifdef F # else # ifdef G # else # ifdef H # else nest = 8; # endif # endif # endif # endif # endif # endif # endif #endif assert( nest == 8); /* 37.5: Nested source file inclusion: at least 8 levels. */ nest = 0; #include "nest1.h" assert( nest == 8); /* 37.6: Parenthesized expression: at least 32 levels. */ #if 0 + (1 - (2 + (3 - (4 + (5 - (6 + (7 - (8 + (9 - (10 + (11 - (12 + \ (13 - (14 + (15 - (16 + (17 - (18 + (19 - (20 + (21 - (22 + (23 - \ (24 + (25 - (26 + (27 - (28 + (29 - (30 + (31 - (32 + 0)))))))))) \ )))))))))))))))))))))) == 0 nest = 32; #endif assert( nest == 32); /* 37.7: Characters in a string (after concatenation): at least 509. */ { char * extremely_long_string1 = "123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 012345678901234567" ; assert( strlen( extremely_long_string1) == 507); } /* 37.8: Characters in a logical source line: at least 509. */ { int a123456789012345678901234567890 = 123450; \ int b123456789012345678901234567890 = 123451; \ int c123456789012345678901234567890 = 123452; \ int d123456789012345678901234567890 = 123453; \ int e123456789012345678901234567890 = 123454; \ int f123456789012345678901234567890 = 123455; \ int A123456789012345678901234567890 = 123456; \ int B123456789012345678901234567890 = 123457; \ int C123456789012345678901234567890 = 123458; \ int D1234567890123456789012 = 123459; assert( a123456789012345678901234567890 == 123450 && D1234567890123456789012 == 123459); } /* 37.9: Macro definitions: at least 1024. */ #include "m1024.h" assert( ZX); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_3_4.c ================================================ /* n_3_4.c: Handling of comment and . */ /* 3.4: Comment and in #error line. */ #error Message of first physical line. \ Message of second physical and first logical line. /* this comment splices the lines */ Message of forth physical and third logical line. ================================================ FILE: tests/test-c/n_4.c ================================================ /* n_4.c: Special tokens. */ #include "defs.h" main( void) { /* 4.1: Digraph spellings in directive line. */ %: define stringize( a) %: a fputs( "started\n", stderr); assert( strcmp( stringize( abc), "abc") == 0); /* 4.2: Digraph spellings are retained in stringization. */ assert( strcmp( stringize( <:), "<" ":") == 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_5.c ================================================ /* n_5.c: Spaces or tabs are allowed at any place in pp-directive line, including between the top of a pp-directive line and '#', and between the '#' and the directive. */ #include "defs.h" main( void) { int abcde = 5; /* |**|[TAB]# |**|[TAB]define |**| MACRO_abcde[TAB]|**| abcde |**| */ /**/ # /**/ define /**/ MACRO_abcde /**/ abcde /**/ fputs( "started\n", stderr); assert( MACRO_abcde == 5); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_6.c ================================================ /* n_6.c: #include directive. */ #include "defs.h" /* 6.1: Header-name quoted by " and " as well as by < and > can include standard headers. */ /* Note: Standard headers can be included any times. */ #include "ctype.h" #include main( void) { int abc = 3; fputs( "started\n", stderr); assert( isalpha( 'a')); /* 6.2: Macro is allowed in #include line. */ #define HEADER "header.h" #include HEADER assert( MACRO_abc == 3); /* 6.3: With macro nonsence but legal. */ #undef MACRO_abc #define ZERO_TOKEN #include ZERO_TOKEN HEADER ZERO_TOKEN assert( MACRO_abc == 3); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_7.c ================================================ /* n_7.c: #line directive. */ #include "defs.h" main( void) { fputs( "started\n", stderr); /* 7.1: Line number and filename. */ #line 1234 "cpp" assert( __LINE__ == 1234); assert( strcmp( __FILE__, "cpp") == 0); /* 7.2: Filename argument is optional. */ #line 2345 assert( __LINE__ == 2345); assert( strcmp( __FILE__, "cpp") == 0); /* 7.3: Argument with macro. */ #define LINE_AND_FILENAME 1234 "n_7.c" #line LINE_AND_FILENAME assert( __LINE__ == 1234); assert( strcmp( __FILE__, "n_7.c") == 0); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_8.c ================================================ /* n_8.c: #error directive. */ /* 8.1: Argument of #error directive is not a subject of macro expansion. Output to stderr as an example: Preprocessing error directive: MACRO is not a positive number. from line 10 of file "n_8.c" */ #define MACRO 0 #if MACRO <= 0 #error MACRO is not a positive number. #endif ================================================ FILE: tests/test-c/n_8_2.c ================================================ /* n_8_2.c: Argument of #error is optional. */ /* 8.2: #error should be executed. */ #error ================================================ FILE: tests/test-c/n_9.c ================================================ /* n_9.c: #pragma directive. */ #include "stdio.h" /* 9.1: Any #pragma directive should be processed or ignored, should not be diagnosed as an error. */ #pragma once #pragma who knows ? main( void) { fputs( "started\n", stderr); fputs( "success\n", stderr); return 0; } ================================================ FILE: tests/test-c/n_i_.lst ================================================ n_1 n_2 n_3 n_4 n_5 n_6 n_7 n_9 n_10 n_11 n_12 n_13 n_13_5 n_13_7 n_13_8 n_13_13 n_15 n_18 n_19 n_20 n_21 n_22 n_23 n_24 n_25 n_26 n_27 n_28 n_29 n_30 n_32 n_37 i_32_3 i_35 i_35_3 ================================================ FILE: tests/test-c/n_std.c ================================================ /* * n_std.c * * 1998/08 made public kmatsui * 2002/08 revised not to conflict with C99 Standard kmatsui * 2004/10 added a few testcases for macro expansion kmatsui * * Samples to test Standard C preprocessing. * This is a strictly-comforming program. * Any Standard-comforming translator must translate successfully this * program. The generated execution program must be executed with the message * on stdout and no other messages on stderr. * A translator must process also #error directive properly, which is not * included here because the directive might cause translator to terminate. */ #include "defs.h" #define ZERO_TOKEN #define TWO_ARGS a,b #define MACRO_0 0 #define MACRO_1 1 #define sub( x, y) (x - y) #define str( a) # a #define xstr( a) str( a) #define glue( a, b) a ## b #define xglue( a, b) glue( a, b) void n_1( void); void n_2( void); void n_3( void); void n_4( void); void n_5( void); void n_6( void); void n_7( void); void n_9( void); void n_10( void); void n_11( void); void n_12( void); void n_13( void); void n_13_5( void); void n_13_7( void); void n_13_8( void); void n_13_13( void); void n_15( void); void n_18( void); void n_19( void); void n_20( void); void n_21( void); void n_22( void); void n_23( void); void n_24( void); void n_25( void); void n_26( void); void n_27( void); void n_28( void); void n_29( void); void n_30( void); void n_32( void); void n_37( void); int main( void) { //n_1(); trigraphs test removed n_2(); n_3(); //n_4(); digraphs test removed n_5(); n_6(); n_7(); n_9(); n_10(); n_11(); n_12(); n_13(); n_13_5(); n_13_7(); n_13_8(); n_13_13(); n_15(); n_18(); n_19(); n_20(); n_21(); n_22(); n_23(); n_24(); n_25(); n_26(); n_27(); n_28(); n_29(); n_30(); n_32(); n_37(); puts( ""); return 0; } char quasi_trigraph[] = { '?', '?', ' ', '?', '?', '?', ' ' , '?', '?', '%', ' ', '?', '?', '^', ' ', '?', '#', '\0' }; void n_2( void) /* Line splicing by sequence. */ { int ab = 1, cd = 2, ef = 3, abcde = 5; /* 2.1: In a #define directive line, between the parameter list and the replacement text. */ #define FUNC( a, b, c) \ a + b + c assert( FUNC( ab, cd, ef) == 6); /* 2.2: In a #define directive line, among the parameter list and among the replacement text. */ #undef FUNC #define FUNC( a, b \ , c) \ a + b \ + c assert (FUNC( ab, cd, ef) == 6); /* 2.3: In a string literal. */ assert (strcmp( "abc\ de", "abcde") == 0); /* 2.4: in midst of an identifier. */ assert( abc\ de == 5); } void n_3( void) /* Handling of comment. */ { int abcd = 4; /* 3.1: A comment is converted to one space. */ assert( strcmp( str( abc/* comment */de), "abc de") == 0); /* 3.2: // is not a comment of C. */ /* assert( strcmp( str( //), "//") == 0); */ /* 3.3: Comment is parsed prior to the parsing of preprocessing directive. */ #if 0 "nonsence"; /* #else still in comment */ #else #define MACRO_abcd /* in comment */ abcd #endif assert( MACRO_abcd == 4); } void n_5( void) /* Spaces or tabs are allowed at any place in pp-directive line, including between the top of a pp-directive line and '#', and between the '#' and the directive. */ { int abcde = 5; /* |**|[TAB]# |**|[TAB]define |**| MACRO_abcde[TAB]|**| abcde |**| */ /**/ # /**/ define /**/ MACRO_abcde /**/ abcde /**/ assert( MACRO_abcde == 5); } /* 6.1: Header-name quoted by " and " as well as by < and > can include standard headers. */ /* Note: Standard headers can be included any times. */ #ifndef NO_SYSTEM_HEADERS #include "ctype.h" #include #endif void n_6( void) /* #include directive. */ { int abc = 3; assert( isalpha( 'a')); /* 6.2: Macro is allowed in #include line. */ #define HEADER "header.h" #include HEADER assert( MACRO_abc == 3); /* 6.3: With macro nonsence but legal. */ #undef MACRO_abc #include ZERO_TOKEN HEADER ZERO_TOKEN assert( MACRO_abc == 3); } void n_7( void) /* #line directive. */ { /* 7.1: Line number and filename. */ #line 1234 "cpp" assert( __LINE__ == 1234); assert( strcmp( __FILE__, "cpp") == 0); /* 7.2: Filename argument is optional. */ #line 2345 assert( __LINE__ == 2345); assert( strcmp( __FILE__, "cpp") == 0); /* 7.3: Argument with macro. */ #define LINE_AND_FILENAME 1234 "n_7.c" #line LINE_AND_FILENAME assert( __LINE__ == 1234); assert( strcmp( __FILE__, "n_7.c") == 0); } /* Restore to correct line number and filename. */ #line 218 "n_std.c" void n_9( void) /* #pragma directive. */ { /* 9.1: Any #pragma directive should be processed or ignored, should not be diagnosed as an error. */ #pragma once #pragma who knows ? } void n_10( void) /* #if, #elif, #else and #endif pp-directive. */ { /* 10.1: */ /* Note: an undefined identifier in #if expression is replaced to 0. */ #if a assert( a); #elif MACRO_0 assert( MACRO_0); #elif MACRO_1 /* Valid block */ assert( MACRO_1); #else assert( 0); #endif /* 10.2: Comments must be processed even if in skipped #if block. */ /* At least tokenization of string literal and character constant is necessary to process comments, e.g. /* is not a comment mark in string literal. */ #ifdef UNDEFINED /* Comment */ "in literal /* is not a comment" #endif } void n_11( void) /* Operator "defined" in #if or #elif directive. */ { int abc = 1, a = 0; /* 11.1: */ #undef MACRO_abc #define MACRO_abc abc #if defined a assert( a); #else assert( MACRO_abc); #endif #if defined (MACRO_abc) assert( MACRO_abc); #else assert( a); #endif /* 11.2: "defined" is an unary operator whose result is 1 or 0. */ #if defined MACRO_0 * 3 != 3 fputs( "Bad handling of defined operator.\n", stderr); #endif #if (!defined ZERO_TOKEN != 0) || (-defined ZERO_TOKEN != -1) fputs( "Bad grouping of defined, -, ! in #if expression.\n", stderr); #endif } #ifndef NO_SYSTEM_HEADERS #include #endif void n_12( void) /* Integer preprocessing number token and type of #if expression. */ { /* 12.1: */ #if LONG_MAX <= LONG_MIN fputs( "Bad evaluation of long.\n", stderr); #endif #if LONG_MAX <= 1073741823 /* 0x3FFFFFFF */ fputs( "Bad evaluation of long.\n", stderr); #endif /* 12.2: */ #if ULONG_MAX / 2 < LONG_MAX fputs( "Bad evaluation of unsigned long.\n", stderr); #endif /* 12.3: Octal number. */ #if 0177777 != 65535 fputs( "Bad evaluation of octal number.\n", stderr); #endif /* 12.4: Hexadecimal number. */ #if 0Xffff != 65535 || 0XFfFf != 65535 fputs( "Bad evaluation of hexadecimal number.\n", stderr); #endif /* 12.5: Suffix 'L' or 'l'. */ #if 0L != 0 || 0l != 0 fputs( "Bad evaluation of 'L' suffix.\n", stderr); #endif /* 12.6: Suffix 'U' or 'u'. */ #if 1U != 1 || 1u != 1 fputs( "Bad evaluation of 'U' suffix.\n", stderr); #endif /* 12.7: Negative integer. */ #if 0 <= -1 fputs( "Bad evaluation of negative number.\n", stderr); #endif } void n_13( void) /* evaluation of #if expressions. */ /* Valid operators are (precedence in this order) : defined, (unary)+, (unary)-, ~, !, *, /, %, +, -, <<, >>, <, >, <=, >=, ==, !=, &, ^, |, &&, ||, ? : */ { /* 13.1: Bit shift. */ #if 1 << 2 != 4 || 8 >> 1 != 4 fputs( "Bad arithmetic of <<, >> operators.\n", stderr); #endif /* 13.2: Bitwise operators. */ #if (3 ^ 5) != 6 || (3 | 5) != 7 || (3 & 5) != 1 fputs( "Bad arithmetic of ^, |, & operators.\n", stderr); #endif /* 13.3: Result of ||, && operators is either of 1 or 0. */ #if (2 || 3) != 1 || (2 && 3) != 1 || (0 || 4) != 1 || (0 && 5) != 0 fputs( "Bad arithmetic of ||, && operators.\n", stderr); #endif /* 13.4: ?, : operator. */ #if (0 ? 1 : 2) != 2 fputs( "Bad arithmetic of ?: operator.\n", stderr); #endif } void n_13_5( void) /* Arithmetic conversion in #if expressions. */ { /* 13.5: The usual arithmetic conversion is not performed on bit shift. */ #if -1 << 3U > 0 fputs( "Bad conversion of bit shift operands.\n", stderr); #endif /* 13.6: Usual arithmetic conversions. */ #if -1 <= 0U /* -1 is converted to unsigned long. */ fputs( "Bad arithmetic conversion.\n", stderr); #endif #if -1 * 1U <= 0 fputs( "Bad arithmetic conversion.\n", stderr); #endif /* Second and third operands of conditional operator are converted to the same type, thus -1 is converted to unsigned long. */ #if (1 ? -1 : 0U) <= 0 fputs( "Bad arithmetic conversion.\n", stderr); #endif } void n_13_7( void) /* Short-circuit evaluation of #if expression. */ { /* 13.7: 10/0 or 10/MACRO_0 are never evaluated, "divide by zero" error cannot occur. */ #if 0 && 10 / 0 #endif #if not_defined && 10 / not_defined #endif #if MACRO_0 && 10 / MACRO_0 > 1 #endif #if MACRO_0 ? 10 / MACRO_0 : 0 #endif #if MACRO_0 == 0 || 10 / MACRO_0 > 1 /* Valid block */ #else #endif } void n_13_8( void) /* Grouping of sub-expressions in #if expression. */ { /* 13.8: Unary operators are grouped from right to left. */ #if (- -1 != 1) || (!!9 != 1) || (-!+!9 != -1) || (~~1 != 1) fputs( "Bad grouping of -, +, !, ~ in #if expression.\n", stderr); #endif /* 13.9: ?: operators are grouped from right to left. */ #if (1 ? 2 ? 3 ? 3 : 2 : 1 : 0) != 3 fputs( "Bad grouping of ? : in #if expression.\n", stderr); #endif /* 13.10: Other operators are grouped from left to right. */ #if (15 >> 2 >> 1 != 1) || (3 << 2 << 1 != 24) fputs( "Bad grouping of >>, << in #if expression.\n", stderr); #endif /* 13.11: Test of precedence. */ #if 3*10/2 >> !0*2 >> !+!-9 != 1 fputs( "Bad grouping of -, +, !, *, /, >> in #if expression.\n", stderr); #endif /* 13.12: Overall test. Grouped as: ((((((+1 - -1 - ~~1 - -!0) & 6) | ((8 % 9) ^ (-2 * -2))) >> 1) == 7) ? 7 : 0) != 7 evaluated to FALSE. */ #if (((+1- -1-~~1- -!0&6|8%9^-2*-2)>>1)==7?7:0)!=7 fputs( "Bad arithmetic of #if expression.\n", stderr); #endif } void n_13_13( void) /* #if expression with macros. */ { #define and && #define or || #define not_eq != #define bitor | /* 13.13: With macros expanding to operators. */ #if (1 bitor 2) == 3 and 4 not_eq 5 or 0 /* #if (1 | 2) == 3 && 4 != 5 || 0 */ #else fputs( "Bad evaluation of macros expanding to operators in #if expression.\n" , stderr); #endif /* 13.14: With macros expanding to 0 token, nonsence but legal. */ #if ZERO_TOKEN MACRO_1 ZERO_TOKEN > ZERO_TOKEN MACRO_0 ZERO_TOKEN /* #if 1 > 0 */ #else fputs( "Bad evaluation of macros expanding to 0 token in #if expression.\n" , stderr); #endif } void n_15( void) /* #ifdef, #ifndef directives. */ { /* 15.1: #ifdef directive. */ #ifdef MACRO_1 /* Valid block */ assert( MACRO_1); #else assert( MACRO_0); #endif /* 15.2: #ifndef directive. */ #ifndef MACRO_1 assert( MACRO_0); #else /* Valid block */ assert( MACRO_1); #endif } void n_18( void) /* #define directive. */ /* Excerpts from ISO C 6.8.3 "Examples". */ #define OBJ_LIKE (1-1) #define FTN_LIKE(a) ( a ) { int c = 3; /* 18.1: Definition of an object-like macro. */ assert( OBJ_LIKE == 0); #ifndef ZERO_TOKEN fputs( "Can't define macro to 0-token.\n", stderr); #endif /* 18.2: Definition of a function-like macro. */ assert( FTN_LIKE( c) == 3); /* 18.3: Spelling in string identical to parameter is not a parameter. */ #define STR( n1, n2) "n1:n2" assert( strcmp( STR( 1, 2), "n1:n2") == 0); } void n_19( void) /* Valid re-definitions of macros. */ { int c = 1; /* 19.1: */ #define OBJ_LIKE /* white space */ (1-1) /* other */ /* 19.2: */ #define FTN_LIKE( a )( /* note the white space */ \ a /* other stuff on this line */ ) assert( FTN_LIKE( c) == 1); } void n_20( void) /* Macro lexically identical to keyword. */ { /* 20.1: */ #define float double float fl; assert( sizeof fl == sizeof (double)); } void n_21( void) /* Tokenization (No preprocessing tokens are merged implicitly). */ { int a = 1, x = 2, y = -3; /* 21.1: */ #define MINUS - assert( -MINUS-a == -1); /* 21.2: */ #undef sub #define sub( a, b) a-b /* '(a)-(b)' is better */ #define Y -y /* '(-y)' is better */ /* x- -y */ assert( sub( x, Y) == -1); } void n_22( void) /* Tokenization of preprocessing number. */ { #define EXP 1 /* 22.1: 12E+EXP is a preprocessing number, EXP is not expanded. */ assert( strcmp( xstr( 12E+EXP), "12E+EXP") == 0); /* 22.2: .2e-EXP is also a preprocessing number. */ assert( strcmp( xstr( .2e-EXP), ".2e-EXP") == 0); /* 22.3: + or - is allowed only following E or e, 12+EXP is not a preprocessing number. */ assert( strcmp( xstr( 12+EXP), "12+1") == 0); } void n_23( void) /* ## operator in macro definition. */ { int xy = 1; /* 23.1: */ assert( glue( x, y) == 1); /* 23.2: Generate a preprocessing number. */ #undef EXP #define EXP 2 assert( xglue( .12e+, EXP) == 12.0); } void n_24( void) /* # operator in macro definition. */ { /* 24.1: */ assert( strcmp( str( a+b), "a+b") == 0); /* 24.2: White spaces between tokens of operand are converted to one space. */ assert( strcmp( str( ab /* comment */ + cd ), "ab + cd") == 0); /* 24.3: \ is inserted before \ and " in or surrounding literals and no other character is inserted to anywhere. */ assert( strcmp( str( '"' + "' \""), "'\"' + \"' \\\"\"") == 0); /* 24.4: Line splicing by is done prior to token parsing. */ assert( strcmp( str( "ab\ c"), "\"abc\"") == 0); /* 24.5: Token separator inserted by macro expansion should be removed. (Meanwhile, tokens should not be merged. See 21.2.) */ #define f(a) a assert( strcmp( xstr( x-f(y)), "x-y") == 0); } void n_25( void) /* Macro arguments are pre-expanded (unless the argument is an operand of # or ## operator) separately, that is, are macro-replaced completely prior to rescanning. */ { int a = 1, b = 2, abc = 3, MACRO_0MACRO_1 = 2; #undef sub #define sub( x, y) (x - y) /* 25.1: "TWO_ARGS" is read as one argument to "sub", then expanded to "a,b", then "x" is substituted by "a,b". */ assert( sub( TWO_ARGS, 1) == 1); /* 25.2: An argument pre-expanded to 0-token. */ assert( sub( ZERO_TOKEN, a) == -1); /* 25.3: "glue( a, b)" is pre-expanded. */ assert( xglue( glue( a, b), c) == 3); /* 25.4: Operands of ## operator are not pre-expanded. */ assert( glue( MACRO_0, MACRO_1) == 2); /* 25.5: Operand of # operator is not pre-expanded. */ assert( strcmp( str( ZERO_TOKEN), "ZERO_TOKEN") == 0); } #undef f #ifdef void int f( a) int a; { return a; } int g( a) int a; { return a * 2; } #else int f( int a) { return a; } int g( int a) { return a * 2; } #endif void n_26( void) /* The name once replaced is not furthur replaced. */ { int x = 1; int AB = 1; int Z[1]; Z[0] = 1; /* 26.1: Directly recursive macro definition. */ /* Z[0]; */ #define Z Z[0] assert( Z == 1); /* 26.2: Intermediately recursive macro definition. */ /* AB; */ #define AB BA #define BA AB assert( AB == 1); /* 26.3: Directly recursive function-like macro definition. */ /* x + f(x); */ #define f(a) a + f(a) assert( f( x) == 2); /* 26.4: Intermediately recursive function-like macro definition. */ /* x + x + g( x); */ #define g(a) a + h( a) #define h(a) a + g( a) assert( g( x) == 4); /* 26.5: Rescanning encounters the non-replaced macro name. */ /* Z[0] + f( Z[0]); */ assert( f( Z) == 2); } void n_27( void) /* Rescanning of a macro raplace any macro call in the replacement text after substitution of parameters by pre-expanded-arguments. This re-examination may involve the succeding sequences from the source file (what a queer thing!). */ { int a = 1, b = 2, c, m = 1, n = 2; /* 27.1: Cascaded use of object-like macros. */ #define NEST8 NEST7 + 8 #define NEST7 NEST6 + 7 #define NEST6 NEST5 + 6 #define NEST5 NEST4 + 5 #define NEST4 NEST3 + 4 #define NEST3 NEST2 + 3 #define NEST2 NEST1 + 2 #define NEST1 1 assert( NEST8 == 36); /* 27.2: Cascaded use of function-like macros. */ #define FUNC4( a, b) FUNC3( a, b) + NEST4 #define FUNC3( a, b) FUNC2( a, b) + NEST3 #define FUNC2( a, b) FUNC1( a, b) + NEST2 #define FUNC1( a, b) (a) + (b) assert( FUNC4( NEST1, NEST2) == 23); /* 27.3: An identifier generated by ## operator is subject to expansion. */ assert( glue( MACRO_, 1) == 1); #define head sub( #define math( op, a, b) op( (a), (b)) /* 27.4: 'sub' as an argument of math() is not pre-expanded, since '(' is missing. */ assert( math( sub, a, b) == -1); /* 27.5: Queer thing. */ c = head a,b ); assert( c == -1); /* 27.6: Recursive macro (the 2nd 'm' is expanded to 'n' since it is in source file). */ #define m n #define n( a) a assert( m( m) == 2); } void n_28( void) /* __FILE__, __LINE__, __DATE__, __TIME__, __STDC__ and __STDC_VERSION are predefined. */ { char * date = __DATE__; /* 28.1: */ assert( strcmp( __FILE__, "n_std.c") == 0); /* 28.2: */ assert( __LINE__ == 751); /* 28.3: */ assert( strlen( __DATE__) == 11); assert( date[ 4] != '0'); /* 28.4: */ assert( strlen( __TIME__) == 8); /* 28.5: */ assert( __STDC__); /* 28.6: */ assert( __STDC_VERSION__ >= 199409L); /* 28.7: */ #include "line.h" } void n_29( void) /* #undef directive. */ { int DEFINED = 1; /* 29.1: Undefined macro is not a macro. */ #define DEFINED #undef DEFINED assert( DEFINED == 1); /* 29.2: Undefining undefined name is not an error. */ #undef UNDEFINED } void n_30( void) /* Macro calls. */ /* Note: Comma separate the arguments of function-like macro call, but comma between matching inner parenthesis doesn't. This feature is tested on so many places in this suite especially on *.c samples which use assert() macro, that no separete item to test this feature is provided. */ { #undef FUNC #define FUNC( a, b, c) a + b + c int a = 1, b = 2, c = 3; /* 30.1: A macro may cross lines. */ assert ( FUNC ( a, b, c ) == 6 ); } void n_32( void) /* Escape sequence in character constant in #if expression. */ { /* 32.1: Character octal escape sequence. */ #if '\123' != 83 fputs( "Bad evaluation of octal escape sequence.\n", stderr); #endif /* 32.2: Character hexadecimal escape sequence. */ #if '\x1b' != '\033' fputs( "Bad evaluation of hexadecimal escape sequence.\n", stderr); #endif } void n_37( void) /* Translation limits. */ { #define MACRO_8 8 /* 37.1: Number of parameters in macro: at least 31. */ #define glue31(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z,A,B,C,D,E) \ a##b##c##d##e##f##g##h##i##j##k##l##m##n##o##p##q##r##s##t##u##v##w##x##y##z##A##B##C##D##E int ABCDEFGHIJKLMNOPQRSTUVWXYZabcde = 31; int ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ = 30; int nest = 0; /* 37.2: Number of arguments in macro call: at least 31. */ assert( glue31( A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R , S, T, U, V, W, X, Y, Z, a, b, c, d, e) == 31); /* 37.3: Significant initial characters in an internal identifier or a macro name: at least 31. */ assert( ABCDEFGHIJKLMNOPQRSTUVWXYZabcd_ == 30); /* 37.4: Nested conditional inclusion: at least 8 levels. */ nest = 0; #ifdef A #else # ifdef B # else # ifdef C # else # ifdef D # else # ifdef E # else # ifdef F # else # ifdef G # else # ifdef H # else nest = 8; # endif # endif # endif # endif # endif # endif # endif #endif assert( nest == 8); /* 37.5: Nested source file inclusion: at least 8 levels. */ nest = 0; #include "nest1.h" assert( nest == 8); /* 37.6: Parenthesized expression: at least 32 levels. */ #if 0 + (1 - (2 + (3 - (4 + (5 - (6 + (7 - (8 + (9 - (10 + (11 - (12 + \ (13 - (14 + (15 - (16 + (17 - (18 + (19 - (20 + (21 - (22 + (23 - \ (24 + (25 - (26 + (27 - (28 + (29 - (30 + (31 - (32 + 0)))))))))) \ )))))))))))))))))))))) == 0 nest = 32; #endif assert( nest == 32); /* 37.7: Characters in a string (after concatenation): at least 509. */ { char * extremely_long_string = "123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 0123456789012345678901234567890123456789012345678901234567890123456789\ 012345678901234567" ; assert( strlen( extremely_long_string) == 507); } /* 37.8: Characters in a logical source line: at least 509. */ { int a123456789012345678901234567890 = 123450; \ int b123456789012345678901234567890 = 123451; \ int c123456789012345678901234567890 = 123452; \ int d123456789012345678901234567890 = 123453; \ int e123456789012345678901234567890 = 123454; \ int f123456789012345678901234567890 = 123455; \ int A123456789012345678901234567890 = 123456; \ int B123456789012345678901234567890 = 123457; \ int C123456789012345678901234567890 = 123458; \ int D1234567890123456789012 = 123459; assert( a123456789012345678901234567890 == 123450 && D1234567890123456789012 == 123459); } /* 37.9: Macro definitions: at least 1024. */ #undef AB #undef BA #undef OR #include "m1024.h" assert( ZX); } ================================================ FILE: tests/test-c/nest1.h ================================================ /* nest1.h */ nest = 1; #include "nest2.h" ================================================ FILE: tests/test-c/nest10.h ================================================ /* nest10.h */ #include "nest11.h" ================================================ FILE: tests/test-c/nest11.h ================================================ /* nest11.h */ #include "nest12.h" ================================================ FILE: tests/test-c/nest12.h ================================================ /* nest12.h */ #include "nest13.h" ================================================ FILE: tests/test-c/nest13.h ================================================ /* nest13.h */ #include "nest14.h" ================================================ FILE: tests/test-c/nest14.h ================================================ /* nest14.h */ #include "nest15.h" ================================================ FILE: tests/test-c/nest15.h ================================================ /* nest15.h */ #ifdef X0F nest = 0x0f; #endif ================================================ FILE: tests/test-c/nest2.h ================================================ /* nest2.h */ nest = 2; #include "nest3.h" ================================================ FILE: tests/test-c/nest3.h ================================================ /* nest3.h */ nest = 3; #include "nest4.h" ================================================ FILE: tests/test-c/nest4.h ================================================ /* nest4.h */ nest = 4; #include "nest5.h" ================================================ FILE: tests/test-c/nest5.h ================================================ /* nest5.h */ nest = 5; #include "nest6.h" ================================================ FILE: tests/test-c/nest6.h ================================================ /* nest6.h */ nest = 6; #include "nest7.h" ================================================ FILE: tests/test-c/nest7.h ================================================ /* nest7.h */ nest = 7; #include "nest8.h" ================================================ FILE: tests/test-c/nest8.h ================================================ /* nest8.h */ #ifndef X0F nest = 8; #else #include "nest9.h" #endif ================================================ FILE: tests/test-c/nest9.h ================================================ /* nest9.h */ #include "nest10.h" ================================================ FILE: tests/test-c/side_cpp ================================================ n_1: Trigraph sequences. n_2: Line splicing by . n_3: Handling of comment. n_4: Tokens spelled by digraphs. n_5: Spaces or tabs in pp-directive. n_6: #include directive. n_7: #line directive. n_9: #pragma directive. n_10: #if, #elif pp-directive. n_11: Operator "defined" in #if. n_12: Pp-number and type of #if expr. n_13: Valid operators in #if expr. n_13.5: Usual arithmetic conversion. n_13.7: Short-circuit evaluation of #if. n_13.8: Grouping of #if sub-expressions. n_13.13: #if expression with macros. n_15: #ifdef, #ifndef directives. n_18: #define directive. n_19: Valid re-definitions of macros. n_20: Macro name identical to keyword. n_21: Tokenization (no token merging). n_22: Tokenization of pp-number. n_23: ## operator in macro definition. n_24: # operator in macro definition. n_25: Pre-expansion of macro args. n_26: No recursive replacement. n_27: Rescanning of a macro. n_28: Standard pre-defined macros. n_29: #undef directive. n_30: Macro call crossing lines. n_32: Escape sequence in char-const. n_37: Translation limits. i_32.3: Character constant in #if. i_35: Multi-character character const. i_35.3: Multi-character wide character. Total number of tests unpassed. ================================================ FILE: tests/test-c/u_1_1.c ================================================ /* u_1_1.c: Undefined behaviors on unterminated line, comment or macro. */ main( void) { /* u.1.1: End of a source file without . */ #include "unbal3.h" int e_1; /* u.1.2: End of a source file with . */ #include "unbal4.h" ; /* u.1.3: End of a source file with an unterminated comment. */ #include "unbal5.h" */ /* u.1.4: End of a source file with an uncompleted macro call. */ #include "unbal6.h" y); return 0; } ================================================ FILE: tests/test-c/u_1_11.c ================================================ /* u_1_11.c: Undefined behaviors on undefined #include syntax or header- name. */ /* u.1.11: Header-name containing ', ", \ or "/*". */ /* Probably illegal filename and fails to open. */ #include "../*line.h" main( void) { /* \ is a legal path-delimiter in MS-DOS or some other OS's. */ #include "..\test-t\line.h" return 0; } ================================================ FILE: tests/test-c/u_1_12.c ================================================ /* u_1_12.c: Undefined behaviors on undefined #include syntax or header- name. */ /* u.1.12: Argument of #include other than header-name. */ #include filename main( void) { return 0; } ================================================ FILE: tests/test-c/u_1_13.c ================================================ /* u_1_13.c: Undefined behaviors on undefined #include syntax or header- name. */ /* u.1.13: Excessive argument in #include directive. */ #include Junk main( void) { return 0; } ================================================ FILE: tests/test-c/u_1_14.c ================================================ /* u_1_14.c: Undefined behaviors on undefined #line syntax. */ main( void) { /* u.1.14: #line directive without an argument of line number. */ #line "filename" /* u.1.15: #line directive with the second argument of other than string literal. */ #line 1234 filename /* u.1.16: Excessive argument in #line directive. */ #line 2345 "filename" Junk return 0; } ================================================ FILE: tests/test-c/u_1_17.c ================================================ /* u_1_17.c: Undefined behaviors on out-of-range #line number. */ #include main( void) { /* u.1.17: Line number argument of #line directive should be in range of [1,32767]. */ #line 32767 /* valid here */ /* line 32767 */ /* line 32768 ? : out of range */ printf( "%d\n", __LINE__); /* 32769 ? or -32767 ?, maybe warned as an out-of-range */ #line 0 #line 32768 /* u.1.18: Line number argument of #line directive should be written in decimal digits. */ #line 0x1000 /* 23, u_1_17.c or other undefined results. */ printf( "%d, %s\n", __LINE__, __FILE__); return 0; } ================================================ FILE: tests/test-c/u_1_19.c ================================================ /* u_1_19.c: Undefined behaviors on undefined #define and #undef syntax. */ #include main( void) { /* u.1.19: A macro expanded to "defined" in #if expression. */ #define DEFINED defined #if DEFINED DEFINED #endif #undef __linux__ #undef __arm__ #define __linux__ 1 #define HAVE_MREMAP defined(__linux__) && !defined(__arm__) /* Wrong macro definition. * This macro should be defined as follows. * #if defined(__linux__) && !defined(__arm__) * #define HAVE_MREMAP 1 * #endif */ #if HAVE_MREMAP mremap(); #endif /* u.1.20: Undefining __FILE__, __LINE__, __DATE__, __TIME__, __STDC__ or "defined" in #undef directive. */ #undef __LINE__ /* 31 or other undefined result. */ printf( "%d\n", __LINE__); /* u.1.21: Defining __FILE__, __LINE__, __DATE__, __TIME__, __STDC__ or "defined" in #define directive. */ #define __LINE__ 1234 /* 37 or other undefined result. */ printf( "%d\n", __LINE__); #define defined defined #if defined defined # error I am not a good preprocessor. #endif return 0; } ================================================ FILE: tests/test-c/u_1_22.c ================================================ /* u_1_22.c: Undefined behaviors on generating invalid pp-token by ## operator. */ #include #define str( a) # a main( void) { /* u.1.22: Result of ## operator is not a valid pp-token. */ #define NUM( dig, exp) dig ## E+ ## exp /* * "E+" is a sequence of two pp-tokens "E" and "+", not a single pp-token. * The first ## concatenates the last pp-token of first argument with "E", * and the second ## concatenates "+" with the first pp-token of the second * argument. * While "12E" (one of the sequence generated by the token concatenation) * is a valid pp-token, "+34" (the another sequence) is not a valid pp-token * and causes an undefined behavior. */ printf( "%e\n", NUM( 12, 34)); return 0; } ================================================ FILE: tests/test-c/u_1_23.c ================================================ /* u_1_23.c: Undefined behaviors on generating invalid pp-token by # operator. */ #include #define str( a) # a main( void) { /* "\\"\""; This sequence is parsed to three tokens "\\" \ "", and will be diagnosed by compiler-proper unless diagnosed by preprocessor. */ puts( str( \"")); return 0; } ================================================ FILE: tests/test-c/u_1_24.c ================================================ /* u_1_24.c: Undefined behaviors on empty argument of macro call. */ /* u.1.24: Empty argument of macro call. */ /* * Note: Since no argument and one empty argument cannot be distinguished * syntactically, additional dummy argument may be necessary for an * intermediate macro to process one empty argument (if possible). */ #include #define ARG( a, dummy) # a #define EMPTY #define SHOWN( n) printf( "%s : %d\n", # n, n) #define SHOWS( s) printf( "%s : %s\n", # s, ARG( s, dummy)) #define add( a, b) (a + b) #define sub( a, b) (a - b) #define math( op, a, b) op( a, b) #define APPEND( a, b) a ## b main( void) { int x = 1; int y = 2; /* printf( "%s : %d\n", "math( sub, , y)", ( - y)); or other undefined behavior. */ SHOWN( math( sub, , y)); /* printf( "%s : %s\n", "EMPTY", ""); or other undefined behavior. */ SHOWS( EMPTY); /* printf( "%s : %s\n", "APPEND( CON, 1)", "CON1"); */ SHOWS( APPEND( CON, 1)); /* printf( "%s : %s\n", "APPEND( CON, )", "CON"); or other undefined behavior. */ SHOWS( APPEND( CON, )); /* printf( "%s : %s\n", "APPEND( , )", ""); or other undefined behavior. */ SHOWS( APPEND( , )); return 0; } ================================================ FILE: tests/test-c/u_1_25.c ================================================ /* u_1_25.c: Undefined behaviors on undefined macro argument. */ #include #define str( a) # a #define sub( x, y) (x - y) #define SUB sub main( void) { int a = 1, b = 2; /* u.1.25: Macro argument otherwise parsed as a directive. */ /* "#define NAME"; or other undefined behaviour. */ puts( str( #define NAME )); #if 0 /* Added by C90: Corrigendum 1 (1994) and deleted by C99 */ /* u.1.26: Expanded macro replacement list end with name of function-like macro. */ SUB( a, b); #endif return 0; } ================================================ FILE: tests/test-c/u_1_27.c ================================================ /* u_1_27.c: Pseudo-directive-line. */ /* u.1.27: Unknown preprocessing directive (other than #pragma). */ #ifdefined MACRO #endif /* The second error. */ main( void) { return 0; } ================================================ FILE: tests/test-c/u_1_28.c ================================================ /* u_1_28.c: Macro expanding to name identical to directive. */ #define D define /* u.1.28: There are following two ways of preprocessing. 1: "D" isn't expanded, because # is the first token of the line. Preprocessor reports that "D" is an unknown directive. 2: "D" is expanded, because that is not a directive. Compiler-phase will diagnose the output of preprocess. Anyway, preprocessor should not interprete this line as a preprocessing directive. */ #D A B main( void) { return 0; } ================================================ FILE: tests/test-c/u_1_5.c ================================================ /* u_1_5.c: Undefined behaviors on illegal characters. */ /* u.1.5: Illegal characters (in other than string literal, character constant, header-name or comment). */ #if 1 ||2 /* 0x01e ^ ^ 0x1f */ #endif /* Maybe the second error. */ /* u.1.6: [VT], [FF] in directive line. */ #if 1 || 2 /* [VT] ^ ^ [FF] */ #endif /* Maybe the second error. */ main( void) { return 0; } ================================================ FILE: tests/test-c/u_1_7_big5.c ================================================ /* u_1_7_big5.c: Invalid multi-byte character sequence (in string literal, character constant, header-name or comment). */ #define str( a) # a #pragma setlocale( "chinese-traditional") /* For Visual C */ #pragma __setlocale( "big5") /* For MCPP */ main( void) { char * cp = str( ""); /* 0xa181 */ return 0; } ================================================ FILE: tests/test-c/u_1_7_eucjp.c ================================================ /* u_1_7_eucjp.c: Invalid multi-byte character sequence (in string literal, character constant, header-name or comment). */ #define str( a) # a #pragma __setlocale( "eucjp") /* For MCPP */ main( void) { char * cp = str( ""); /* 0xb1a0 */ return 0; } ================================================ FILE: tests/test-c/u_1_7_gb.c ================================================ /* u_1_7_gb.c: Invalid multi-byte character sequence (in string literal, character constant, header-name or comment). */ #define str( a) # a #pragma setlocale( "chinese-simplified") /* For Visual C */ #pragma __setlocale( "gb2312") /* For MCPP */ main( void) { char * cp = str( ""); /* 0xb1a0 */ return 0; } ================================================ FILE: tests/test-c/u_1_7_jis.c ================================================ /* u_1_7_jis.c: Invalid multi-byte character sequence (in string literal, character constant, header-name or comment). */ #define str( a) # a #pragma __setlocale( "jis") /* For MCPP */ main( void) { char * cp = str( "$B1 (B"); /* 0x3120 */ return 0; } ================================================ FILE: tests/test-c/u_1_7_ksc.c ================================================ /* u_1_7_ksc.c: Invalid multi-byte character sequence (in string literal, character constant, header-name or comment). */ #define str( a) # a #pragma setlocale( "korean") /* For Visual C */ #pragma __setlocale( "ksc5601") /* For MCPP */ main( void) { char * cp = str( ""); /* 0xb1a0 */ return 0; } ================================================ FILE: tests/test-c/u_1_7_sjis.c ================================================ /* u_1_7_sjis.c: Invalid multi-byte character sequence (in string literal, character constant, header-name or comment). */ #define str( a) # a #pragma setlocale( "japanese") /* For Visual C */ #pragma __setlocale( "sjis") /* For MCPP */ main( void) { char * cp = str( "8"); /* 0x9138 */ return 0; } ================================================ FILE: tests/test-c/u_1_7_utf8.c ================================================ /* u_1_7_utf8.c: Invalid multi-byte character sequence (in string literal, character constant, header-name or comment). */ #define str( a) # a #pragma __setlocale( "utf8") /* For MCPP */ main( void) { char * cp = str( "字"); /* 0xe5ad97 : legal */ char * ecp1 = str( ""); /* 0xc0af : overlong */ char * ecp2 = str( ""); /* 0xe09fbf : overlong */ char * ecp3 = str( ""); /* 0xeda080 : UTF-16 surrogate */ return 0; } ================================================ FILE: tests/test-c/u_1_8.c ================================================ /* u_1_8.c: Undefined behaviors on unterminated quotations. */ /* u.1.8: Unterminated character constant. */ /* The following "comment" may not interpreted as a comment but swallowed by the unterminated character constant. */ #error I can't understand. /* Token error prior to execution of #error. */ main( void) { /* u.1.9: Unterminated string literal. */ char * string = "String literal across the lines. " ; return 0; } /* u.1.10: Unterminated header-name. */ #include #define str( a) # a main( void) { int a = 1, b = 2; char * string; /* u.1.1: End of a source file without . */ #include "unbal3.h" int e_1; /* u.1.2: End of a source file with . */ #include "unbal4.h" ; /* u.1.3: End of a source file with an unterminated comment. */ #include "unbal5.h" */ /* u.1.4: End of a source file with an uncompleted macro call. */ #include "unbal6.h" y); /* u.1.5: Illegal characters (in other than string literal, character constant, header-name or comment). */ #if 1 ||2 /* 0x01e ^ ^ 0x1f */ #endif /* Maybe the second error. */ /* u.1.6: [VT], [FF] in directive line. */ #if 1 || 2 /* [VT] ^ ^ [FF] */ #endif /* Maybe the second error. */ /* u.1.7: Invalid multi-byte character sequence (in string literal, character constant, header-name or comment). */ #if '== 0x8e /* ^ 0x8e */ #endif /* Maybe the second error. */ /* u.1.8: Unterminated character constant. */ /* The following "comment" may not interpreted as a comment but swallowed by the unterminated character constant. */ #error I can't understand. /* Token error prior to execution of #error. */ /* u.1.9: Unterminated string literal. */ string = "String literal across the lines. " ; /* u.1.10: Unterminated header-name. */ #include Junk /* u.1.14: #line directive without an argument of line number. */ #line "filename" /* u.1.15: #line directive with the second argument of other than string literal. */ #line 1234 filename /* u.1.16: Excessive argument in #line directive. */ #line 2345 "filename" Junk /* 90, undefs.c or other undefined results. */ printf( "%d, %s\n", __LINE__, __FILE__); /* u.1.17: Line number argument of #line directive should be in range of [1,32767]. */ #line 32767 /* valid here */ /* line 32767 */ /* line 32768 ? : out of range */ printf( "%d\n", __LINE__); /* 32769 ? or -32767 ?, maybe warned as an out-of-range */ #line 0 #line 32768 /* u.1.18: Line number argument of #line directive should be written in decimal digits. */ #line 0x1000 #line 108 /* Restore to correct line number. */ /* u.1.19: A macro expanded to "defined" in #if expression. */ #define DEFINED defined #if DEFINED DEFINED #endif #undef __linux__ #undef __arm__ #define __linux__ 1 #define HAVE_MREMAP defined(__linux__) && !defined(__arm__) /* Wrong macro definition. * This macro should be defined as follows. * #if defined(__linux__) && !defined(__arm__) * #define HAVE_MREMAP 1 * #endif */ #if HAVE_MREMAP mremap(); #endif /* u.1.20: Undefining __FILE__, __LINE__, __DATE__, __TIME__, __STDC__, __STDC_VERSION__ or "defined" in #undef directive. */ #undef __LINE__ /* 131 or other undefined result. */ printf( "%d\n", __LINE__); /* u.1.21: Defining __FILE__, __LINE__, __DATE__, __TIME__, __STDC__, __STDC_VERSION__ or "defined" in #define directive. */ #define __LINE__ 1234 /* 137 or other undefined result. */ printf( "%d\n", __LINE__); #define defined defined #if defined defined # error I am not a good preprocessor. #endif /* u.1.22: Result of ## operator is not a valid pp-token. */ #define NUM( dig, exp) dig ## E+ ## exp /* * "E+" is a sequence of two pp-tokens "E" and "+", not a single pp-token. * The first ## concatenates the last pp-token of first argument with "E", * and the second ## concatenates "+" with the first pp-token of the second * argument. * While "12E" (one of the sequence generated by the token concatenation) * is a valid pp-token, "+34" (the another sequence) is not a valid pp-token * and causes an undefined behavior. */ printf( "%e\n", NUM( 12, 34)); /* u.1.23: Result of # operator is not a valid string literal. */ /* "\\"\""; This sequence is parsed to three tokens "\\" \ "", and will be diagnosed by compiler-proper unless diagnosed by preprocessor. */ puts( str( \"")); /* u.1.24: Empty argument of macro call. */ /* * Note: Since no argument and one empty argument cannot be distinguished * syntactically, additional dummy argument may be necessary for an * intermediate macro to process one empty argument (if possible). */ #define ARG( a, dummy) # a #define EMPTY #define SHOWN( n) printf( "%s : %d\n", # n, n) #define SHOWS( s) printf( "%s : %s\n", # s, ARG( s, dummy)) #define add( a, b) (a + b) #define sub( a, b) (a - b) #define math( op, a, b) op( a, b) #define APPEND( a, b) a ## b /* printf( "%s : %d\n", "math( sub, , y)", ( - y)); or other undefined behavior. */ SHOWN( math( sub, , y)); /* printf( "%s : %s\n", "EMPTY", ""); or other undefined behavior. */ SHOWS( EMPTY); /* printf( "%s : %s\n", "APPEND( CON, 1)", "CON1"); */ SHOWS( APPEND( CON, 1)); /* printf( "%s : %s\n", "APPEND( CON, )", "CON"); or other undefined behavior. */ SHOWS( APPEND( CON, )); /* printf( "%s : %s\n", "APPEND( , )", ""); or other undefined behavior. */ SHOWS( APPEND( , )); /* u.1.25: Macro argument otherwise parsed as a directive. */ /* "#define NAME"; or other undefined behaviour. */ puts( str( #define NAME )); #define sub( x, y) (x - y) #define SUB sub #if 0 /* Added by C90: Corrigendum 1 (1994) and deleted by C99 */ /* u.1.26: Expanded macro replacement list end with name of function-like macro. */ SUB( a, b); #endif /* u.1.27: Unknown preprocessing directive (other than #pragma). */ #ifdefined MACRO #endif /* The second error. */ #define D define /* u.1.28: There are following two ways of preprocessing. 1: "D" isn't expanded, because # is the first token of the line. Preprocessor reports that "D" is an unknown directive. 2: "D" is expanded, because that is not a directive. Compiler-phase will diagnose the output of preprocess. Anyway, preprocessor should not interprete this line as a preprocessing directive. */ #D A B /* u.2.1: Undefined escape sequence. */ #if '\x' #endif /* u.2.2: Illegal bit shift count. */ #if 1 << -1 #endif #if 1 << 64 #endif return 0; } ================================================ FILE: tests/test-c/unspcs.c ================================================ /* * unspcs.c: * 1998/08 made public kmatsui * * These texts are unportable ones, because the order of the evaluation is * unspecified. Good preprocessor will warn at these texts even if the * results are valid. Good preprocessor will also document the order of * evaluation and the behavior on invalid results. * Note: Order of evaluation of sub-expressions (other than operands of &&, * ||, ? :) of #if expression is also unspecified. The order, however, never * affects the result, because #if expression never cause side effect, so no * warning is necessary. Precedence and grouping rules of operators are other * things than order of evaluation, and shall be obeyed by preprocessor. */ #include "defs.h" #define str( a) # a #define xstr( a) str( a) main( void) { /* s.1.1: Order of evaluation of #, ## operators. */ #define MAKWIDESTR( s) L ## # s /* Either of L"name"; or L# name; ("L#" is not a valid pp-token). */ assert( MAKWIDESTR( name)[ 0] == L'n'); /* s.1.2: Order of evaluation of ## operators. */ #define glue3( a, b, c) a ## b ## c /* "1.a" or undefined, since .a is not a valid pp-token, while 1. and 1.a are valid pp-tokens. */ puts( xstr( glue3( 1, ., a))); return 0; } ================================================ FILE: tests/test-c/warns.c ================================================ /* * warns.c: * 1998/08 made public kmatsui * 2002/12 slightly modified kmatsui * 2003/11 added a few samples kmatsui */ /* * The following texts are legal but suspicious ones. Good preprocessor * will warn at these texts. */ #include "defs.h" main( void) { /* w.1.1: "/*" in comment. */ /* comment /* nested comment and no closing */ /* w.1.2: Rescanning of replacement text involves succeding text. */ #define sub( x, y) (x - y) #define head sub( int a = 1, b = 2, c; c = head a,b ); assert( c == -1); #define OBJECT_LIKE FUNCTION_LIKE #define FUNCTION_LIKE( x, y) (x + y) c = OBJECT_LIKE( a, b); assert( c == 3); /* w.2.1: Negative number converted to positive in #if expression. */ #if -1 < 0U #endif /* w.2.2: Out of range of unsigned type (wraps around and never overflow) in #if expression. */ #if 0U - 1 #endif /* * The following texts are legal but non-portable ones, since these requires * translation limits greater than the minima quaranteed by C90. Good * preprocessor will warn at these texts (at least when user wants), unless * it diagnose these as errors. */ { int nest; int A0B0C0D0E0F0G0H0O0O1O2O3P0P1P2 = 123; /* w.3.1: Number of parameters in macro: more than 31. */ #define glue63( \ a0, b0, c0, d0, e0, f0, g0, h0, i0, j0, k0, l0, m0, n0, o0, p0, \ a1, b1, c1, d1, e1, f1, g1, h1, i1, j1, k1, l1, m1, n1, o1, p1, \ a2, b2, c2, d2, e2, f2, g2, h2, i2, j2, k2, l2, m2, n2, o2, p2, \ a3, b3, c3, d3, e3, f3, g3, h3, i3, j3, k3, l3, m3, n3, o3) \ a0 ## b0 ## c0 ## d0 ## e0 ## f0 ## g0 ## h0 ## \ o0 ## o1 ## o2 ## o3 ## p0 ## p1 ## p2 /* w.3.2: Number of arguments in macro call: more than 31. */ assert( glue63( A0, B0, C0, D0, E0, F0, G0, H0, I0, J0, K0, L0, M0, N0, O0, P0, A1, B1, C1, D1, E1, F1, G1, H1, I1, J1, K1, L1, M1, N1, O1, P1, A2, B2, C2, D2, E2, F2, G2, H2, I2, J2, K2, L2, M2, N2, O2, P2, A3, B3, C3, D3, E3, F3, G3, H3, I3, J3, K3, L3, M3, N3, O3) == 123); /* w.3.3: Initial significant characters in an identifier: more than 31. */ { int A23456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef = 63; assert( A23456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef == 63); } /* w.3.4: Nested conditional inclusion: more than 8 levels. */ nest = 0; #define X0F #include "ifdef15.h" assert( nest == 0x0f); /* w.3.5: Nested source file inclusion: more than 8 levels. */ nest = 0; #define X0F #include "nest1.h" assert( nest == 0x0f); /* w.3.6: Parenthesized expression: more than 32 levels. */ nest = 0; #if \ (0x00 + (0x01 - (0x02 + (0x03 - (0x04 + (0x05 - (0x06 + (0x07 - \ (0x08 + (0x09 - (0x0A + (0x0B - (0x0C + (0x0D - (0x0E + (0x0F - \ (0x10 + (0x11 - (0x12 + (0x13 - (0x14 + (0x15 - (0x16 + (0x17 - \ (0x18 + (0x19 - (0x1A + (0x1B - (0x1C + (0x1D - (0x1E + (0x1F - \ (0x20 + (0x21 - (0x22 + (0x23 - (0x24 + (0x25 - (0x26 + (0x27 - \ (0x28 + (0x29 - (0x2A + (0x2B - (0x2C + (0x2D - (0x2E + (0x2F - \ (0x30 + (0x31 - (0x32 + (0x33 - (0x34 + (0x35 - (0x36 + (0x37 - \ (0x38 + (0x39 - (0x3A + (0x3B - (0x3C + (0x3D - 0x3E) \ ))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))) \ == -1 nest = 63; #endif assert( nest == 63); } /* w.3.7: Characters in a string (after concatenation): more than 509. */ { char *string1023 = "123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ 1123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ 2123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ 3123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ 4123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ 5123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ 6123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ 7123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ 8123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ 9123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ a123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ b123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ c123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ d123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ e123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\ f123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" ; assert( strlen( string1023) == 1023); } /* w.3.8: Characters in a logical source line: more than 509. */ { int a123456789012345678901234567890 = 123450; \ int b123456789012345678901234567890 = 123451; \ int c123456789012345678901234567890 = 123452; \ int d123456789012345678901234567890 = 123453; \ int e123456789012345678901234567890 = 123454; \ int f123456789012345678901234567890 = 123455; \ int g123456789012345678901234567890 = 123456; \ int h123456789012345678901234567890 = 123457; \ int i123456789012345678901234567890 = 123458; \ int j123456789012345678901234567890 = 123459; \ int k123456789012345678901234567890 = 123460; \ int l123456789012345678901234567890 = 123461; \ int m123456789012345678901234567890 = 123462; \ int n123456789012345678901234567890 = 123463; \ int o123456789012345678901234567890 = 123464; \ int p123456789012345678901234567890 = 123465; \ int q123456789012345678901234567890 = 123466; \ int r123456789012345678901234567890 = 123467; \ int s123456789012345678901234567890 = 123468; \ int t1234567890123456 = 123469; } /* w.3.9: Macro definitions: more than 1024 (including predefined ones). */ #include "m1024.h" assert( ZX == 1); return 0; }