[
  {
    "path": ".doctrine-project.json",
    "content": "{\n    \"active\": true,\n    \"name\": \"Lexer\",\n    \"slug\": \"lexer\",\n    \"docsSlug\": \"doctrine-lexer\",\n    \"versions\": [\n        {\n            \"name\": \"3.1\",\n            \"branchName\": \"3.1.x\",\n            \"slug\": \"latest\",\n            \"upcoming\": true\n        },\n        {\n            \"name\": \"3.0\",\n            \"branchName\": \"3.0.x\",\n            \"slug\": \"3.1\",\n            \"current\": true\n        },\n        {\n            \"name\": \"2.1\",\n            \"branchName\": \"2.1.x\",\n            \"slug\": \"2.1\",\n            \"maintained\": false\n        },\n        {\n            \"name\": \"2.0\",\n            \"branchName\": \"2.0.x\",\n            \"slug\": \"2.0\",\n            \"maintained\": false\n        },\n        {\n            \"name\": \"1.2\",\n            \"branchName\": \"1.2.x\",\n            \"slug\": \"1.2\",\n            \"maintained\": false\n        },\n        {\n            \"name\": \"1.1\",\n            \"branchName\": \"1.1\",\n            \"slug\": \"1.1\",\n            \"maintained\": false\n        },\n        {\n            \"name\": \"1.0\",\n            \"branchName\": \"1.0\",\n            \"slug\": \"1.0\",\n            \"maintained\": false\n        }\n    ]\n}\n"
  },
  {
    "path": ".gitattributes",
    "content": "# Auto-detect text files, ensure they use LF.\n* text=auto eol=lf\n\n # Exclude non-essential files from dist\n/.github export-ignore\n/docs export-ignore\n/tests export-ignore\n/.doctrine-project.json export-ignore\n/.gitattributes export-ignore\n/.gitignore export-ignore\n/phpcs.xml.dist export-ignore\n/phpstan.neon.dist export-ignore\n/phpunit.xml.dist export-ignore\n"
  },
  {
    "path": ".github/FUNDING.yml",
    "content": "patreon: phpdoctrine\ntidelift: packagist/doctrine%2Flexer\ncustom: https://www.doctrine-project.org/sponsorship.html\n"
  },
  {
    "path": ".github/dependabot.yml",
    "content": "version: 2\nupdates:\n  - package-ecosystem: \"github-actions\"\n    directory: \"/\"\n    schedule:\n      interval: \"weekly\"\n    labels:\n      - \"CI\"\n"
  },
  {
    "path": ".github/workflows/coding-standards.yml",
    "content": "\nname: \"Coding Standards\"\n\non:\n  pull_request:\n    branches:\n      - \"*.x\"\n    paths:\n      - .github/workflows/coding-standards.yml\n      - composer.*\n      - src/**\n      - phpcs.xml.dist\n      - tests/**\n  push:\n    branches:\n      - \"*.x\"\n    paths:\n      - .github/workflows/coding-standards.yml\n      - composer.*\n      - src/**\n      - phpcs.xml.dist\n      - tests/**\n\njobs:\n  coding-standards:\n    name: \"Coding Standards\"\n    uses: \"doctrine/.github/.github/workflows/coding-standards.yml@14.0.0\"\n"
  },
  {
    "path": ".github/workflows/composer-lint.yml",
    "content": "name: \"Composer Lint\"\n\non:\n  pull_request:\n    branches:\n      - \"*.x\"\n    paths:\n      - .github/workflows/composer-lint.yml\n      - composer.json\n  push:\n    branches:\n      - \"*.x\"\n    paths:\n      - .github/workflows/composer-lint.yml\n      - composer.json\n\njobs:\n  composer-lint:\n    name: \"Composer Lint\"\n    uses: \"doctrine/.github/.github/workflows/composer-lint.yml@13.1.0\"\n"
  },
  {
    "path": ".github/workflows/continuous-integration.yml",
    "content": "\nname: \"Continuous Integration\"\n\non:\n  pull_request:\n    branches:\n      - \"*.x\"\n    paths:\n      - .github/workflows/continuous-integration.yml\n      - composer.*\n      - src/**\n      - phpunit.xml.dist\n      - tests/**\n  push:\n    branches:\n      - \"*.x\"\n    paths:\n      - .github/workflows/continuous-integration.yml\n      - composer.*\n      - src/**\n      - phpunit.xml.dist\n      - tests/**\n\njobs:\n  phpunit:\n    name: \"PHPUnit\"\n    uses: \"doctrine/.github/.github/workflows/continuous-integration.yml@14.0.0\"\n    with:\n      php-versions: '[\"8.1\", \"8.2\", \"8.3\", \"8.4\", \"8.5\"]'\n    secrets:\n      CODECOV_TOKEN: \"${{ secrets.CODECOV_TOKEN }}\"\n"
  },
  {
    "path": ".github/workflows/release-on-milestone-closed.yml",
    "content": "name: \"Automatic Releases\"\n\non:\n  milestone:\n    types:\n      - \"closed\"\n\njobs:\n  release:\n    name: \"Git tag, release & create merge-up PR\"\n    uses: \"doctrine/.github/.github/workflows/release-on-milestone-closed.yml@14.0.0\"\n    secrets:\n      GIT_AUTHOR_EMAIL: ${{ secrets.GIT_AUTHOR_EMAIL }}\n      GIT_AUTHOR_NAME: ${{ secrets.GIT_AUTHOR_NAME }}\n      ORGANIZATION_ADMIN_TOKEN: ${{ secrets.ORGANIZATION_ADMIN_TOKEN }}\n      SIGNING_SECRET_KEY: ${{ secrets.SIGNING_SECRET_KEY }}\n"
  },
  {
    "path": ".github/workflows/static-analysis.yml",
    "content": "\nname: \"Static Analysis\"\n\non:\n  pull_request:\n    branches:\n      - \"*.x\"\n    paths:\n      - .github/workflows/static-analysis.yml\n      - composer.*\n      - src/**\n      - phpstan*\n      - tests/**\n  push:\n    branches:\n      - \"*.x\"\n    paths:\n      - .github/workflows/static-analysis.yml\n      - composer.*\n      - src/**\n      - phpstan*\n      - tests/**\n\njobs:\n  static-analysis:\n    name: \"Static Analysis\"\n    uses: \"doctrine/.github/.github/workflows/phpstan.yml@14.0.0\"\n"
  },
  {
    "path": ".github/workflows/website-schema.yml",
    "content": "\nname: \"Website config validation\"\n\non:\n  pull_request:\n    branches:\n      - \"*.x\"\n    paths:\n      - \".doctrine-project.json\"\n      - \".github/workflows/website-schema.yml\"\n  push:\n    branches:\n      - \"*.x\"\n    paths:\n      - \".doctrine-project.json\"\n      - \".github/workflows/website-schema.yml\"\n\njobs:\n  json-validate:\n    name: \"Validate JSON schema\"\n    uses: \"doctrine/.github/.github/workflows/website-schema.yml@14.0.0\"\n"
  },
  {
    "path": ".gitignore",
    "content": "/vendor\n/composer.lock\n/phpunit.xml\n/.phpunit.cache\n/phpcs.xml\n/.phpcs-cache\n"
  },
  {
    "path": "LICENSE",
    "content": "Copyright (c) 2006-2018 Doctrine Project\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\nof the Software, and to permit persons to whom the Software is furnished to do\nso, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
  },
  {
    "path": "README.md",
    "content": "# Doctrine Lexer\n\n[![Build Status](https://github.com/doctrine/lexer/workflows/Continuous%20Integration/badge.svg)](https://github.com/doctrine/lexer/actions)\n\nBase library for a lexer that can be used in Top-Down, Recursive Descent Parsers.\n\nThis lexer is used in Doctrine Annotations and in Doctrine ORM (DQL).\n\nhttps://www.doctrine-project.org/projects/lexer.html\n"
  },
  {
    "path": "UPGRADE.md",
    "content": "Note about upgrading: Doctrine uses static and runtime mechanisms to raise\nawareness about deprecated code.\n\n- Use of `@deprecated` docblock that is detected by IDEs (like PHPStorm) or\n  Static Analysis tools (like Psalm, phpstan)\n- Use of our low-overhead runtime deprecation API, details:\n  https://github.com/doctrine/deprecations/\n\n# Upgrade to 3.0.0\n\n`Doctrine\\Common\\Lexer\\Token` no longer implements `ArrayAccess`.\nParameter type declarations have been added to\n`Doctrine\\Common\\Lexer\\AbstractLexer` and `Doctrine\\Common\\Lexer\\Token`.\nYou should add both parameter type declarations and return type declarations to\nyour lexers, based on the `@return` phpdoc.\n\n# Upgrade to 2.0.0\n\n`AbstractLexer::glimpse()` and `AbstractLexer::peek()` now return\ninstances of `Doctrine\\Common\\Lexer\\Token`, which is an array-like class\nUsing it as an array is deprecated in favor of using properties of that class.\nUsing `count()` on it is deprecated with no replacement.\n"
  },
  {
    "path": "composer.json",
    "content": "{\n    \"name\": \"doctrine/lexer\",\n    \"description\": \"PHP Doctrine Lexer parser library that can be used in Top-Down, Recursive Descent Parsers.\",\n    \"license\": \"MIT\",\n    \"type\": \"library\",\n    \"keywords\": [\n        \"php\",\n        \"parser\",\n        \"lexer\",\n        \"annotations\",\n        \"docblock\"\n    ],\n    \"authors\": [\n        {\n            \"name\": \"Guilherme Blanco\",\n            \"email\": \"guilhermeblanco@gmail.com\"\n        },\n        {\n            \"name\": \"Roman Borschel\",\n            \"email\": \"roman@code-factory.org\"\n        },\n        {\n            \"name\": \"Johannes Schmitt\",\n            \"email\": \"schmittjoh@gmail.com\"\n        }\n    ],\n    \"homepage\": \"https://www.doctrine-project.org/projects/lexer.html\",\n    \"require\": {\n        \"php\": \"^8.1\"\n    },\n    \"require-dev\": {\n        \"doctrine/coding-standard\": \"^14\",\n        \"phpstan/phpstan\": \"^2\",\n        \"phpunit/phpunit\": \"^10.5.58 || ^12.5.4\"\n    },\n    \"autoload\": {\n        \"psr-4\": {\n            \"Doctrine\\\\Common\\\\Lexer\\\\\": \"src\"\n        }\n    },\n    \"autoload-dev\": {\n        \"psr-4\": {\n            \"Doctrine\\\\Tests\\\\Common\\\\Lexer\\\\\": \"tests\"\n        }\n    },\n    \"config\": {\n        \"allow-plugins\": {\n            \"composer/package-versions-deprecated\": true,\n            \"dealerdirect/phpcodesniffer-composer-installer\": true\n        },\n        \"sort-packages\": true\n    }\n}\n"
  },
  {
    "path": "docs/en/dql-parser.rst",
    "content": "DQL Lexer\n=========\n\nHere is a more complicated example from the Doctrine ORM project.\nThe ``Doctrine\\ORM\\Query\\Lexer`` implementation for DQL looks something\nlike the following:\n\n.. code-block:: php\n\n    <?php\n\n    use Doctrine\\Common\\Lexer\\AbstractLexer;\n\n    class Lexer extends AbstractLexer\n    {\n        // All tokens that are not valid identifiers must be < 100\n        public const T_NONE              = 1;\n        public const T_INTEGER           = 2;\n        public const T_STRING            = 3;\n        public const T_INPUT_PARAMETER   = 4;\n        public const T_FLOAT             = 5;\n        public const T_CLOSE_PARENTHESIS = 6;\n        public const T_OPEN_PARENTHESIS  = 7;\n        public const T_COMMA             = 8;\n        public const T_DIVIDE            = 9;\n        public const T_DOT               = 10;\n        public const T_EQUALS            = 11;\n        public const T_GREATER_THAN      = 12;\n        public const T_LOWER_THAN        = 13;\n        public const T_MINUS             = 14;\n        public const T_MULTIPLY          = 15;\n        public const T_NEGATE            = 16;\n        public const T_PLUS              = 17;\n        public const T_OPEN_CURLY_BRACE  = 18;\n        public const T_CLOSE_CURLY_BRACE = 19;\n\n        // All tokens that are identifiers or keywords that could be considered as identifiers should be >= 100\n        public const T_ALIASED_NAME         = 100;\n        public const T_FULLY_QUALIFIED_NAME = 101;\n        public const T_IDENTIFIER           = 102;\n\n        // All keyword tokens should be >= 200\n        public const T_ALL      = 200;\n        public const T_AND      = 201;\n        public const T_ANY      = 202;\n        public const T_AS       = 203;\n        public const T_ASC      = 204;\n        public const T_AVG      = 205;\n        public const T_BETWEEN  = 206;\n        public const T_BOTH     = 207;\n        public const T_BY       = 208;\n        public const T_CASE     = 209;\n        public const T_COALESCE = 210;\n        public const T_COUNT    = 211;\n        public const T_DELETE   = 212;\n        public const T_DESC     = 213;\n        public const T_DISTINCT = 214;\n        public const T_ELSE     = 215;\n        public const T_EMPTY    = 216;\n        public const T_END      = 217;\n        public const T_ESCAPE   = 218;\n        public const T_EXISTS   = 219;\n        public const T_FALSE    = 220;\n        public const T_FROM     = 221;\n        public const T_GROUP    = 222;\n        public const T_HAVING   = 223;\n        public const T_HIDDEN   = 224;\n        public const T_IN       = 225;\n        public const T_INDEX    = 226;\n        public const T_INNER    = 227;\n        public const T_INSTANCE = 228;\n        public const T_IS       = 229;\n        public const T_JOIN     = 230;\n        public const T_LEADING  = 231;\n        public const T_LEFT     = 232;\n        public const T_LIKE     = 233;\n        public const T_MAX      = 234;\n        public const T_MEMBER   = 235;\n        public const T_MIN      = 236;\n        public const T_NEW      = 237;\n        public const T_NOT      = 238;\n        public const T_NULL     = 239;\n        public const T_NULLIF   = 240;\n        public const T_OF       = 241;\n        public const T_OR       = 242;\n        public const T_ORDER    = 243;\n        public const T_OUTER    = 244;\n        public const T_PARTIAL  = 245;\n        public const T_SELECT   = 246;\n        public const T_SET      = 247;\n        public const T_SOME     = 248;\n        public const T_SUM      = 249;\n        public const T_THEN     = 250;\n        public const T_TRAILING = 251;\n        public const T_TRUE     = 252;\n        public const T_UPDATE   = 253;\n        public const T_WHEN     = 254;\n        public const T_WHERE    = 255;\n        public const T_WITH     = 256;\n\n        /**\n         * Creates a new query scanner object.\n         *\n         * @param string $input A query string.\n         */\n        public function __construct(string $input)\n        {\n            $this->setInput($input);\n        }\n\n        /**\n         * {@inheritdoc}\n         */\n        protected function getCatchablePatterns(): array\n        {\n            return [\n                '[a-z_][a-z0-9_]*\\:[a-z_][a-z0-9_]*(?:\\\\\\[a-z_][a-z0-9_]*)*', // aliased name\n                '[a-z_\\\\\\][a-z0-9_]*(?:\\\\\\[a-z_][a-z0-9_]*)*', // identifier or qualified name\n                '(?:[0-9]+(?:[\\.][0-9]+)*)(?:e[+-]?[0-9]+)?', // numbers\n                \"'(?:[^']|'')*'\", // quoted strings\n                '\\?[0-9]*|:[a-z_][a-z0-9_]*', // parameters\n            ];\n        }\n\n        /**\n         * {@inheritdoc}\n         */\n        protected function getNonCatchablePatterns(): array\n        {\n            return ['\\s+', '(.)'];\n        }\n\n        /**\n         * {@inheritdoc}\n         */\n        protected function getType(&$value): int\n        {\n            $type = self::T_NONE;\n\n            switch (true) {\n                // Recognize numeric values\n                case (is_numeric($value)):\n                    if (strpos($value, '.') !== false || stripos($value, 'e') !== false) {\n                        return self::T_FLOAT;\n                    }\n\n                    return self::T_INTEGER;\n\n                // Recognize quoted strings\n                case ($value[0] === \"'\"):\n                    $value = str_replace(\"''\", \"'\", substr($value, 1, strlen($value) - 2));\n\n                    return self::T_STRING;\n\n                // Recognize identifiers, aliased or qualified names\n                case (ctype_alpha($value[0]) || $value[0] === '_' || $value[0] === '\\\\'):\n                    $name = 'Doctrine\\ORM\\Query\\Lexer::T_' . strtoupper($value);\n\n                    if (defined($name)) {\n                        $type = constant($name);\n\n                        if ($type > 100) {\n                            return $type;\n                        }\n                    }\n\n                    if (strpos($value, ':') !== false) {\n                        return self::T_ALIASED_NAME;\n                    }\n\n                    if (strpos($value, '\\\\') !== false) {\n                        return self::T_FULLY_QUALIFIED_NAME;\n                    }\n\n                    return self::T_IDENTIFIER;\n\n                // Recognize input parameters\n                case ($value[0] === '?' || $value[0] === ':'):\n                    return self::T_INPUT_PARAMETER;\n\n                // Recognize symbols\n                case ($value === '.'):\n                    return self::T_DOT;\n                case ($value === ','):\n                    return self::T_COMMA;\n                case ($value === '('):\n                    return self::T_OPEN_PARENTHESIS;\n                case ($value === ')'):\n                    return self::T_CLOSE_PARENTHESIS;\n                case ($value === '='):\n                    return self::T_EQUALS;\n                case ($value === '>'):\n                    return self::T_GREATER_THAN;\n                case ($value === '<'):\n                    return self::T_LOWER_THAN;\n                case ($value === '+'):\n                    return self::T_PLUS;\n                case ($value === '-'):\n                    return self::T_MINUS;\n                case ($value === '*'):\n                    return self::T_MULTIPLY;\n                case ($value === '/'):\n                    return self::T_DIVIDE;\n                case ($value === '!'):\n                    return self::T_NEGATE;\n                case ($value === '{'):\n                    return self::T_OPEN_CURLY_BRACE;\n                case ($value === '}'):\n                    return self::T_CLOSE_CURLY_BRACE;\n\n                // Default\n                default:\n                    // Do nothing\n            }\n\n            return $type;\n        }\n    }\n\nThis is roughly what the DQL Parser looks like that uses the above\nLexer implementation:\n\n.. note::\n\n    You can see the full implementation `here <https://github.com/doctrine/doctrine2/blob/1.3.x/lib/Doctrine/ORM/Query/Parser.php>`_.\n\n.. code-block:: php\n\n    <?php\n\n    class Parser\n    {\n        private $lexer;\n\n        public function __construct(string $dql)\n        {\n            $this->lexer = new Lexer();\n            $this->lexer->setInput($dql);\n        }\n\n        // ...\n\n        public function getAST(): AST\\SelectStatement|AST\\UpdateStatement|AST\\DeleteStatement\n        {\n            // Parse & build AST\n            $AST = $this->QueryLanguage();\n\n            // ...\n\n            return $AST;\n        }\n\n        public function QueryLanguage(): AST\\SelectStatement|AST\\UpdateStatement|AST\\DeleteStatement\n        {\n            $this->lexer->moveNext();\n\n            switch ($this->lexer->lookahead->type) {\n                case Lexer::T_SELECT:\n                    $statement = $this->SelectStatement();\n                    break;\n                case Lexer::T_UPDATE:\n                    $statement = $this->UpdateStatement();\n                    break;\n                case Lexer::T_DELETE:\n                    $statement = $this->DeleteStatement();\n                    break;\n                default:\n                    $this->syntaxError('SELECT, UPDATE or DELETE');\n            }\n\n            // Check for end of string\n            if ($this->lexer->lookahead !== null) {\n                $this->syntaxError('end of string');\n            }\n\n            return $statement;\n        }\n\n        // ...\n    }\n\nNow the AST is used to transform the DQL query in to portable SQL for whatever relational\ndatabase you are using!\n\n.. code-block:: php\n\n    <?php\n\n    $parser = new Parser('SELECT u FROM User u');\n    $AST = $parser->getAST(); // returns \\Doctrine\\ORM\\Query\\AST\\SelectStatement\n\nWhat is an AST?\n===============\n\nAST stands for `Abstract syntax tree <http://en.wikipedia.org/wiki/Abstract_syntax_tree>`_.\nIn computer science, an abstract syntax tree (AST), or just syntax tree, is a\ntree representation of the abstract syntactic structure of source code written\nin a programming language. Each node of the tree denotes a construct occurring in\nthe source code.\n"
  },
  {
    "path": "docs/en/index.rst",
    "content": "Introduction\n============\n\nDoctrine Lexer is a library that can be used in Top-Down, Recursive\nDescent Parsers. This lexer is used in Doctrine Annotations and in\nDoctrine ORM (DQL).\n\nTo write your own parser you just need to extend ``Doctrine\\Common\\Lexer\\AbstractLexer``\nand implement the following three abstract methods.\n\n.. code-block:: php\n\n    /**\n     * Lexical catchable patterns.\n     *\n     * @return string[]\n     */\n    abstract protected function getCatchablePatterns();\n\n    /**\n     * Lexical non-catchable patterns.\n     *\n     * @return string[]\n     */\n    abstract protected function getNonCatchablePatterns();\n\n    /** Retrieve token type. Also processes the token value if necessary. */\n    abstract protected function getType(string &$value): int;\n\nThese methods define the `lexical <http://en.wikipedia.org/wiki/Lexical_analysis>`_\ncatchable and non-catchable patterns and a method for returning the\ntype of a token and filtering the value if necessary.\n\nThe Lexer is responsible for giving you an API to walk across a\nstring one character at a time and analyze the type of each character, value and position of\neach token in the string. The low level API of the lexer is pretty simple:\n\n- ``setInput($input)`` - Sets the input data to be tokenized. The Lexer is immediately reset and the new input tokenized.\n- ``reset()`` - Resets the lexer.\n- ``resetPeek()`` - Resets the peek pointer to 0.\n- ``resetPosition($position = 0)`` - Resets the lexer position on the input to the given position.\n- ``isNextToken($token)`` - Checks whether a given token matches the current lookahead.\n- ``isNextTokenAny(array $tokens)`` - Checks whether any of the given tokens matches the current lookahead.\n- ``moveNext()`` - Moves to the next token in the input string.\n- ``skipUntil($type)`` - Tells the lexer to skip input tokens until it sees a token with the given value.\n- ``isA($value, $token)`` - Checks if given value is identical to the given token.\n- ``peek()`` - Moves the lookahead token forward.\n- ``glimpse()`` - Peeks at the next token, returns it and immediately resets the peek.\n"
  },
  {
    "path": "docs/en/sidebar.rst",
    "content": ".. toctree::\n    :depth: 3\n\n    index\n    simple-parser-example\n    dql-parser\n"
  },
  {
    "path": "docs/en/simple-parser-example.rst",
    "content": "Simple Parser Example\n=====================\n\nExtend the ``Doctrine\\Common\\Lexer\\AbstractLexer`` class and implement\nthe ``getCatchablePatterns``, ``getNonCatchablePatterns``, and ``getType``\nmethods. Here is a very simple example lexer implementation named ``CharacterTypeLexer``.\nIt tokenizes a string to ``T_UPPER``, ``T_LOWER`` and``T_NUMBER`` tokens:\n\n.. code-block:: php\n    <?php\n\n    use Doctrine\\Common\\Lexer\\AbstractLexer;\n\n    /**\n     * @extends AbstractLexer<CharacterTypeLexer::T_*, string>\n     */\n    class CharacterTypeLexer extends AbstractLexer\n    {\n        const T_UPPER =  1;\n        const T_LOWER =  2;\n        const T_NUMBER = 3;\n\n        protected function getCatchablePatterns(): array\n        {\n            return [\n                '[a-zA-Z0-9]',\n            ];\n        }\n\n        protected function getNonCatchablePatterns(): array\n        {\n            return [];\n        }\n\n        protected function getType(&$value): int\n        {\n            if (is_numeric($value)) {\n                return self::T_NUMBER;\n            }\n\n            if (strtoupper($value) === $value) {\n                return self::T_UPPER;\n            }\n\n            if (strtolower($value) === $value) {\n                return self::T_LOWER;\n            }\n        }\n    }\n\nUse ``CharacterTypeLexer`` to extract an array of upper case characters:\n\n.. code-block:: php\n    <?php\n\n    class UpperCaseCharacterExtracter\n    {\n        public function __construct(private CharacterTypeLexer $lexer)\n        {\n        }\n\n        /** @return list<string> */\n        public function getUpperCaseCharacters(string $string): array\n        {\n            $this->lexer->setInput($string);\n            $this->lexer->moveNext();\n\n            $upperCaseChars = [];\n            while (true) {\n                if (!$this->lexer->lookahead) {\n                    break;\n                }\n\n                $this->lexer->moveNext();\n\n                if ($this->lexer->token->isA(CharacterTypeLexer::T_UPPER)) {\n                    $upperCaseChars[] = $this->lexer->token->value;\n                }\n            }\n\n            return $upperCaseChars;\n        }\n    }\n\n    $upperCaseCharacterExtractor = new UpperCaseCharacterExtracter(new CharacterTypeLexer());\n    $upperCaseCharacters = $upperCaseCharacterExtractor->getUpperCaseCharacters('1aBcdEfgHiJ12');\n\n    print_r($upperCaseCharacters);\n\nThe variable ``$upperCaseCharacters`` contains all of the upper case\ncharacters:\n\n.. code-block:: php\n    Array\n    (\n        [0] => B\n        [1] => E\n        [2] => H\n        [3] => J\n    )\n\nThis is a simple example but it should demonstrate the low level API\nthat can be used to build more complex parsers.\n"
  },
  {
    "path": "phpcs.xml.dist",
    "content": "<?xml version=\"1.0\"?>\n<ruleset\n        xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n        xsi:noNamespaceSchemaLocation=\"vendor/squizlabs/php_codesniffer/phpcs.xsd\"\n>\n    <arg name=\"basepath\" value=\".\"/>\n    <arg name=\"extensions\" value=\"php\"/>\n    <arg name=\"parallel\" value=\"80\"/>\n    <arg name=\"cache\" value=\".phpcs-cache\"/>\n    <arg name=\"colors\" />\n\n    <config name=\"php_version\" value=\"80100\"/>\n\n    <!-- Ignore warnings and show progress of the run -->\n    <arg value=\"nps\"/>\n\n    <file>src</file>\n    <file>tests</file>\n\n    <rule ref=\"Doctrine\">\n        <!-- Will cause BC breaks to method signatures - disabled for now -->\n        <exclude name=\"SlevomatCodingStandard.TypeHints.ReturnTypeHint.MissingNativeTypeHint\" />\n\n        <!-- Disabled to avoid class renaming - to be handled in a separate PR -->\n        <exclude name=\"SlevomatCodingStandard.Classes.SuperfluousAbstractClassNaming\" />\n    </rule>\n</ruleset>\n"
  },
  {
    "path": "phpstan.neon.dist",
    "content": "parameters:\n  level: max\n  paths:\n    - %rootDir%/../../../src\n    - %rootDir%/../../../tests\n"
  },
  {
    "path": "phpunit.xml.dist",
    "content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\n<!-- https://phpunit.de/manual/current/en/appendixes.configuration.html -->\n<phpunit xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n         xsi:noNamespaceSchemaLocation=\"vendor/phpunit/phpunit/phpunit.xsd\"\n         backupGlobals=\"false\"\n         colors=\"true\"\n         bootstrap=\"vendor/autoload.php\"\n         cacheDirectory=\".phpunit.cache\"\n>\n    <php>\n      <ini name=\"error_reporting\" value=\"-1\" />\n    </php>\n\n    <testsuites>\n        <testsuite name=\"Doctrine lexer Test Suite\">\n            <directory>tests</directory>\n        </testsuite>\n    </testsuites>\n\n    <source>\n        <include>\n            <directory suffix=\".php\">src</directory>\n        </include>\n    </source>\n</phpunit>\n"
  },
  {
    "path": "src/AbstractLexer.php",
    "content": "<?php\n\ndeclare(strict_types=1);\n\nnamespace Doctrine\\Common\\Lexer;\n\nuse ReflectionClass;\nuse UnitEnum;\n\nuse function implode;\nuse function preg_split;\nuse function sprintf;\nuse function substr;\n\nuse const PREG_SPLIT_DELIM_CAPTURE;\nuse const PREG_SPLIT_NO_EMPTY;\nuse const PREG_SPLIT_OFFSET_CAPTURE;\n\n/**\n * Base class for writing simple lexers, i.e. for creating small DSLs.\n *\n * @template T of UnitEnum|string|int\n * @template V of string|int|float|bool\n */\nabstract class AbstractLexer\n{\n    /**\n     * Lexer original input string.\n     */\n    private string $input;\n\n    /**\n     * Array of scanned tokens.\n     *\n     * @var list<Token<T, V>>\n     */\n    private array $tokens = [];\n\n    /**\n     * Current lexer position in input string.\n     */\n    private int $position = 0;\n\n    /**\n     * Current peek of current lexer position.\n     */\n    private int $peek = 0;\n\n    /**\n     * The next token in the input.\n     *\n     * @var Token<T, V>|null\n     */\n    public Token|null $lookahead;\n\n    /**\n     * The last matched/seen token.\n     *\n     * @var Token<T, V>|null\n     */\n    public Token|null $token;\n\n    /**\n     * Composed regex for input parsing.\n     *\n     * @var non-empty-string|null\n     */\n    private string|null $regex = null;\n\n    /**\n     * Sets the input data to be tokenized.\n     *\n     * The Lexer is immediately reset and the new input tokenized.\n     * Any unprocessed tokens from any previous input are lost.\n     *\n     * @param string $input The input to be tokenized.\n     *\n     * @return void\n     */\n    public function setInput(string $input)\n    {\n        $this->input  = $input;\n        $this->tokens = [];\n\n        $this->reset();\n        $this->scan($input);\n    }\n\n    /**\n     * Resets the lexer.\n     *\n     * @return void\n     */\n    public function reset()\n    {\n        $this->lookahead = null;\n        $this->token     = null;\n        $this->peek      = 0;\n        $this->position  = 0;\n    }\n\n    /**\n     * Resets the peek pointer to 0.\n     *\n     * @return void\n     */\n    public function resetPeek()\n    {\n        $this->peek = 0;\n    }\n\n    /**\n     * Resets the lexer position on the input to the given position.\n     *\n     * @param int $position Position to place the lexical scanner.\n     *\n     * @return void\n     */\n    public function resetPosition(int $position = 0)\n    {\n        $this->position = $position;\n    }\n\n    /**\n     * Retrieve the original lexer's input until a given position.\n     *\n     * @return string\n     */\n    public function getInputUntilPosition(int $position)\n    {\n        return substr($this->input, 0, $position);\n    }\n\n    /**\n     * Checks whether a given token matches the current lookahead.\n     *\n     * @param T $type\n     *\n     * @return bool\n     *\n     * @phpstan-assert-if-true !=null $this->lookahead\n     */\n    public function isNextToken(int|string|UnitEnum $type)\n    {\n        return $this->lookahead !== null && $this->lookahead->isA($type);\n    }\n\n    /**\n     * Checks whether any of the given tokens matches the current lookahead.\n     *\n     * @param list<T> $types\n     *\n     * @return bool\n     *\n     * @phpstan-assert-if-true !=null $this->lookahead\n     */\n    public function isNextTokenAny(array $types)\n    {\n        return $this->lookahead !== null && $this->lookahead->isA(...$types);\n    }\n\n    /**\n     * Moves to the next token in the input string.\n     *\n     * @return bool\n     *\n     * @phpstan-impure\n     * @phpstan-assert-if-true !null $this->lookahead\n     */\n    public function moveNext()\n    {\n        $this->peek      = 0;\n        $this->token     = $this->lookahead;\n        $this->lookahead = isset($this->tokens[$this->position])\n            ? $this->tokens[$this->position++] : null;\n\n        return $this->lookahead !== null;\n    }\n\n    /**\n     * Tells the lexer to skip input tokens until it sees a token with the given value.\n     *\n     * @param T $type The token type to skip until.\n     *\n     * @return void\n     */\n    public function skipUntil(int|string|UnitEnum $type)\n    {\n        while ($this->lookahead !== null && ! $this->lookahead->isA($type)) {\n            $this->moveNext();\n        }\n    }\n\n    /**\n     * Checks if given value is identical to the given token.\n     *\n     * @return bool\n     */\n    public function isA(string $value, int|string|UnitEnum $token)\n    {\n        return $this->getType($value) === $token;\n    }\n\n    /**\n     * Moves the lookahead token forward.\n     *\n     * @return Token<T, V>|null The next token or NULL if there are no more tokens ahead.\n     */\n    public function peek()\n    {\n        if (isset($this->tokens[$this->position + $this->peek])) {\n            return $this->tokens[$this->position + $this->peek++];\n        }\n\n        return null;\n    }\n\n    /**\n     * Peeks at the next token, returns it and immediately resets the peek.\n     *\n     * @return Token<T, V>|null The next token or NULL if there are no more tokens ahead.\n     *\n     * @phpstan-impure\n     */\n    public function glimpse()\n    {\n        $peek       = $this->peek();\n        $this->peek = 0;\n\n        return $peek;\n    }\n\n    /**\n     * Scans the input string for tokens.\n     *\n     * @param string $input A query string.\n     *\n     * @return void\n     */\n    protected function scan(string $input)\n    {\n        if (! isset($this->regex)) {\n            $this->regex = sprintf(\n                '/(%s)|%s/%s',\n                implode(')|(', $this->getCatchablePatterns()),\n                implode('|', $this->getNonCatchablePatterns()),\n                $this->getModifiers(),\n            );\n        }\n\n        $flags   = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;\n        $matches = preg_split($this->regex, $input, -1, $flags);\n\n        if ($matches === false) {\n            // Work around https://bugs.php.net/78122\n            $matches = [[$input, 0]];\n        }\n\n        foreach ($matches as $match) {\n            // Must remain before 'value' assignment since it can change content\n            $firstMatch = $match[0];\n            $type       = $this->getType($firstMatch);\n\n            $this->tokens[] = new Token(\n                $firstMatch,\n                $type,\n                $match[1],\n            );\n        }\n    }\n\n    /**\n     * Gets the literal for a given token.\n     *\n     * @param T $token\n     *\n     * @return int|string\n     */\n    public function getLiteral(int|string|UnitEnum $token)\n    {\n        if ($token instanceof UnitEnum) {\n            return $token::class . '::' . $token->name;\n        }\n\n        $className = static::class;\n\n        $reflClass = new ReflectionClass($className);\n        $constants = $reflClass->getConstants();\n\n        foreach ($constants as $name => $value) {\n            if ($value === $token) {\n                return $className . '::' . $name;\n            }\n        }\n\n        return $token;\n    }\n\n    /**\n     * Regex modifiers\n     *\n     * @return string\n     */\n    protected function getModifiers()\n    {\n        return 'iu';\n    }\n\n    /**\n     * Lexical catchable patterns.\n     *\n     * @return string[]\n     */\n    abstract protected function getCatchablePatterns();\n\n    /**\n     * Lexical non-catchable patterns.\n     *\n     * @return string[]\n     */\n    abstract protected function getNonCatchablePatterns();\n\n    /**\n     * Retrieve token type. Also processes the token value if necessary.\n     *\n     * @return T|null\n     *\n     * @param-out V $value\n     */\n    abstract protected function getType(string &$value);\n}\n"
  },
  {
    "path": "src/Token.php",
    "content": "<?php\n\ndeclare(strict_types=1);\n\nnamespace Doctrine\\Common\\Lexer;\n\nuse UnitEnum;\n\nuse function in_array;\n\n/**\n * @template T of UnitEnum|string|int\n * @template-covariant V of string|int|float|bool\n */\nfinal class Token\n{\n    /**\n     * The string value of the token in the input string\n     *\n     * @readonly\n     * @var V\n     */\n    public string|int|float|bool $value;\n\n    /**\n     * The type of the token (identifier, numeric, string, input parameter, none)\n     *\n     * @readonly\n     * @var T|null\n     */\n    public $type;\n\n    /**\n     * The position of the token in the input string\n     *\n     * @readonly\n     */\n    public int $position;\n\n    /**\n     * @param V      $value\n     * @param T|null $type\n     */\n    public function __construct(string|int|float|bool $value, $type, int $position)\n    {\n        $this->value    = $value;\n        $this->type     = $type;\n        $this->position = $position;\n    }\n\n    /** @param T ...$types */\n    public function isA(...$types): bool\n    {\n        return in_array($this->type, $types, true);\n    }\n}\n"
  },
  {
    "path": "tests/AbstractLexerTest.php",
    "content": "<?php\n\ndeclare(strict_types=1);\n\nnamespace Doctrine\\Tests\\Common\\Lexer;\n\nuse Doctrine\\Common\\Lexer\\AbstractLexer;\nuse Doctrine\\Common\\Lexer\\Token;\nuse PHPUnit\\Framework\\Attributes\\DataProvider;\nuse PHPUnit\\Framework\\TestCase;\n\nuse function array_map;\nuse function assert;\nuse function count;\nuse function is_int;\nuse function is_numeric;\nuse function setlocale;\n\nuse const LC_ALL;\n\nclass AbstractLexerTest extends TestCase\n{\n    private ConcreteLexer $concreteLexer;\n\n    public function setUp(): void\n    {\n        $this->concreteLexer = new ConcreteLexer();\n    }\n\n    public function tearDown(): void\n    {\n        setlocale(LC_ALL, null);\n    }\n\n    /** @phpstan-return list<array{string, list<Token<string, string|int>>}> */\n    public static function dataProvider(): array\n    {\n        return [\n            [\n                'price=10',\n                [\n                    new Token('price', 'string', 0),\n                    new Token('=', 'operator', 5),\n                    new Token(10, 'int', 6),\n                ],\n            ],\n        ];\n    }\n\n    public function testResetPeek(): void\n    {\n        $expectedTokens = [\n            new Token('price', 'string', 0),\n            new Token('=', 'operator', 5),\n            new Token(10, 'int', 6),\n        ];\n\n        $this->concreteLexer->setInput('price=10');\n\n        $this->assertEquals($expectedTokens[0], $this->concreteLexer->peek());\n        $this->assertEquals($expectedTokens[1], $this->concreteLexer->peek());\n        $this->concreteLexer->resetPeek();\n        $this->assertEquals($expectedTokens[0], $this->concreteLexer->peek());\n    }\n\n    public function testResetPosition(): void\n    {\n        $expectedTokens = [\n            new Token('price', 'string', 0),\n            new Token('=', 'operator', 5),\n            new Token(10, 'int', 6),\n        ];\n\n        $this->concreteLexer->setInput('price=10');\n        $this->assertNull($this->concreteLexer->lookahead);\n\n        $this->assertTrue($this->concreteLexer->moveNext());\n        $this->assertEquals($expectedTokens[0], $this->concreteLexer->lookahead);\n\n        $this->assertTrue($this->concreteLexer->moveNext());\n        $this->assertEquals($expectedTokens[1], $this->concreteLexer->lookahead);\n\n        $this->concreteLexer->resetPosition(0);\n\n        $this->assertTrue($this->concreteLexer->moveNext());\n        $this->assertEquals($expectedTokens[0], $this->concreteLexer->lookahead);\n    }\n\n    /** @phpstan-param list<Token<string, string|int>>  $expectedTokens */\n    #[DataProvider('dataProvider')]\n    public function testMoveNext(string $input, array $expectedTokens): void\n    {\n        $this->concreteLexer->setInput($input);\n        $this->assertNull($this->concreteLexer->lookahead);\n\n        for ($i = 0; $i < count($expectedTokens); $i++) {\n            $this->assertTrue($this->concreteLexer->moveNext());\n            $this->assertEquals($expectedTokens[$i], $this->concreteLexer->lookahead);\n        }\n\n        $this->assertFalse($this->concreteLexer->moveNext());\n        $this->assertNull($this->concreteLexer->lookahead);\n    }\n\n    public function testSkipUntil(): void\n    {\n        $this->concreteLexer->setInput('price=10');\n\n        $this->assertTrue($this->concreteLexer->moveNext());\n        $this->concreteLexer->skipUntil('operator');\n\n        $this->assertEquals(\n            new Token('=', 'operator', 5),\n            $this->concreteLexer->lookahead,\n        );\n    }\n\n    public function testUtf8Mismatch(): void\n    {\n        $this->concreteLexer->setInput(\"\\xE9=10\");\n\n        $this->assertTrue($this->concreteLexer->moveNext());\n\n        $this->assertEquals(\n            new Token(\"\\xE9=10\", 'string', 0),\n            $this->concreteLexer->lookahead,\n        );\n    }\n\n    /** @phpstan-param list<Token<string, string|int>> $expectedTokens */\n    #[DataProvider('dataProvider')]\n    public function testPeek(string $input, array $expectedTokens): void\n    {\n        $this->concreteLexer->setInput($input);\n        foreach ($expectedTokens as $expectedToken) {\n            $actualToken = $this->concreteLexer->peek();\n            assert($actualToken !== null);\n            $this->assertEquals($expectedToken, $actualToken);\n            $this->assertSame($expectedToken->value, $actualToken->value);\n            $this->assertSame($expectedToken->type, $actualToken->type);\n            $this->assertSame($expectedToken->position, $actualToken->position);\n        }\n\n        $this->assertNull($this->concreteLexer->peek());\n    }\n\n    /** @phpstan-param list<Token<string, string|int>> $expectedTokens */\n    #[DataProvider('dataProvider')]\n    public function testGlimpse(string $input, array $expectedTokens): void\n    {\n        $this->concreteLexer->setInput($input);\n\n        foreach ($expectedTokens as $expectedToken) {\n            $actualToken = $this->concreteLexer->glimpse();\n            assert($actualToken !== null);\n            $this->assertEquals($expectedToken, $actualToken);\n            $this->assertEquals($expectedToken, $this->concreteLexer->glimpse());\n            $this->assertSame($expectedToken->value, $actualToken->value);\n            $this->assertSame($expectedToken->type, $actualToken->type);\n            $this->assertSame($expectedToken->position, $actualToken->position);\n            $this->concreteLexer->moveNext();\n        }\n\n        $this->assertNull($this->concreteLexer->peek());\n    }\n\n    /** @phpstan-return list<array{string, int, string}> */\n    public static function inputUntilPositionDataProvider(): array\n    {\n        return [\n            ['price=10', 5, 'price'],\n        ];\n    }\n\n    #[DataProvider('inputUntilPositionDataProvider')]\n    public function testGetInputUntilPosition(\n        string $input,\n        int $position,\n        string $expectedInput,\n    ): void {\n        $this->concreteLexer->setInput($input);\n\n        $this->assertSame($expectedInput, $this->concreteLexer->getInputUntilPosition($position));\n    }\n\n    /** @phpstan-param list<Token<string, string|int>> $expectedTokens */\n    #[DataProvider('dataProvider')]\n    public function testIsNextToken(string $input, array $expectedTokens): void\n    {\n        $this->concreteLexer->setInput($input);\n\n        $this->concreteLexer->moveNext();\n        for ($i = 0; $i < count($expectedTokens); $i++) {\n            assert($expectedTokens[$i]->type !== null);\n            $this->assertTrue($this->concreteLexer->isNextToken($expectedTokens[$i]->type));\n            $this->concreteLexer->moveNext();\n        }\n    }\n\n    /** @phpstan-param list<Token<string, string|int>> $expectedTokens */\n    #[DataProvider('dataProvider')]\n    public function testIsNextTokenAny(string $input, array $expectedTokens): void\n    {\n        $allTokenTypes = array_map(static function ($token): string {\n            assert($token->type !== null);\n\n            return $token->type;\n        }, $expectedTokens);\n\n        $this->concreteLexer->setInput($input);\n\n        $this->concreteLexer->moveNext();\n        for ($i = 0; $i < count($expectedTokens); $i++) {\n            assert($expectedTokens[$i]->type !== null);\n            $this->assertTrue($this->concreteLexer->isNextTokenAny([$expectedTokens[$i]->type]));\n            $this->assertTrue($this->concreteLexer->isNextTokenAny($allTokenTypes));\n            $this->concreteLexer->moveNext();\n        }\n    }\n\n    public function testGetLiteral(): void\n    {\n        $this->assertSame('Doctrine\\Tests\\Common\\Lexer\\ConcreteLexer::INT', $this->concreteLexer->getLiteral('int'));\n        $this->assertSame('fake_token', $this->concreteLexer->getLiteral('fake_token'));\n    }\n\n    public function testGetLiteralWithEnumLexer(): void\n    {\n        $enumLexer = new EnumLexer();\n        $this->assertSame(\n            'Doctrine\\Tests\\Common\\Lexer\\TokenType::OPERATOR',\n            $enumLexer->getLiteral(TokenType::OPERATOR),\n        );\n    }\n\n    public function testIsA(): void\n    {\n        $this->assertTrue($this->concreteLexer->isA('11', 'int'));\n        $this->assertTrue($this->concreteLexer->isA('1.1', 'int'));\n        $this->assertTrue($this->concreteLexer->isA('=', 'operator'));\n        $this->assertTrue($this->concreteLexer->isA('>', 'operator'));\n        $this->assertTrue($this->concreteLexer->isA('<', 'operator'));\n        $this->assertTrue($this->concreteLexer->isA('fake_text', 'string'));\n    }\n\n    public function testAddCatchablePatternsToMutableLexer(): void\n    {\n        $mutableLexer = new MutableLexer();\n        $mutableLexer->addCatchablePattern('[a-z]');\n        $mutableLexer->setInput('one');\n        $token = $mutableLexer->glimpse();\n\n        $this->assertNotNull($token);\n        $this->assertEquals('o', $token->value);\n\n        $mutableLexer = new MutableLexer();\n        $mutableLexer->addCatchablePattern('[a-z]+');\n        $mutableLexer->setInput('one');\n        $token = $mutableLexer->glimpse();\n\n        $this->assertNotNull($token);\n        $this->assertEquals('one', $token->value);\n    }\n\n    public function testMarkerAnnotationLocaleTr(): void\n    {\n        setlocale(LC_ALL, 'tr_TR.utf8', 'tr_TR');\n        $mutableLexer = new MutableLexer();\n        $mutableLexer->addCatchablePattern('[a-z_\\\\\\][a-z0-9_\\:\\\\\\]*[a-z_][a-z0-9_]*');\n        $mutableLexer->addCatchablePattern('(?:[+-]?[0-9]+(?:[\\.][0-9]+)*)(?:[eE][+-]?[0-9]+)?');\n        $mutableLexer->addCatchablePattern('\"(?:\"\"|[^\"])*+\"');\n        $mutableLexer->setInput('@ODM\\Id');\n\n        self::assertNull($mutableLexer->token);\n        self::assertNull($mutableLexer->lookahead);\n        self::assertTrue($mutableLexer->moveNext());\n        self::assertNull($mutableLexer->token);\n        self::assertNotNull($mutableLexer->lookahead);\n        self::assertEquals('@', $mutableLexer->lookahead->value);\n        self::assertTrue($mutableLexer->moveNext());\n        self::assertNotNull($mutableLexer->token);\n        self::assertEquals('@', $mutableLexer->token->value);\n        self::assertEquals('ODM\\Id', $mutableLexer->lookahead->value);\n    }\n\n    public function testCanTokenizeFloatValue(): void\n    {\n        $lexer = new /** @template-extends AbstractLexer<int, string|int|float|bool> */ class () extends AbstractLexer {\n            final public const T_NONE    = 1;\n            final public const T_INTEGER = 2;\n            final public const T_FLOAT   = 4;\n            final public const T_BOOL    = 8;\n\n            protected function getType(string|int|float|bool &$value): int\n            {\n                if ($value === 'y') {\n                    $value = true;\n\n                    return self::T_BOOL;\n                }\n\n                if (is_numeric($value)) {\n                    $value += 0;\n\n                    if (is_int($value)) {\n                        return self::T_INTEGER;\n                    }\n\n                    return self::T_FLOAT;\n                }\n\n                return self::T_NONE;\n            }\n\n            /** {@inheritDoc} */\n            protected function getCatchablePatterns(): array\n            {\n                return [\n                    '(?:[0-9]+)(?:[\\.][0-9]+)?(?:e[+-]?[0-9]+)?',\n                    'y',\n                ];\n            }\n\n            /** {@inheritDoc} */\n            protected function getNonCatchablePatterns(): array\n            {\n                return ['\\s+'];\n            }\n        };\n\n        $lexer->setInput('123.456');\n        $token = $lexer->peek();\n        assert($token !== null);\n        self::assertSame(123.456, $token->value, 'expect a real float, not a numerical string');\n\n        $lexer->setInput('y');\n        $token = $lexer->peek();\n        assert($token !== null);\n        self::assertTrue($token->value, 'expect a real bool, not a numerical string');\n    }\n}\n"
  },
  {
    "path": "tests/ConcreteLexer.php",
    "content": "<?php\n\ndeclare(strict_types=1);\n\nnamespace Doctrine\\Tests\\Common\\Lexer;\n\nuse Doctrine\\Common\\Lexer\\AbstractLexer;\n\nuse function in_array;\nuse function is_numeric;\n\n/** @extends AbstractLexer<string, string|int> */\nclass ConcreteLexer extends AbstractLexer\n{\n    final public const INT = 'int';\n\n    /**\n     * {@inheritDoc}\n     */\n    protected function getCatchablePatterns(): array\n    {\n        return [\n            '=|<|>',\n            '[a-z]+',\n            '\\d+',\n        ];\n    }\n\n    /**\n     * {@inheritDoc}\n     */\n    protected function getNonCatchablePatterns(): array\n    {\n        return [\n            '\\s+',\n            '(.)',\n        ];\n    }\n\n    protected function getType(string|int|float &$value): string\n    {\n        if (is_numeric($value)) {\n            $value = (int) $value;\n\n            return 'int';\n        }\n\n        if (in_array($value, ['=', '<', '>'])) {\n            return 'operator';\n        }\n\n        return 'string';\n    }\n}\n"
  },
  {
    "path": "tests/EnumLexer.php",
    "content": "<?php\n\ndeclare(strict_types=1);\n\nnamespace Doctrine\\Tests\\Common\\Lexer;\n\nuse Doctrine\\Common\\Lexer\\AbstractLexer;\n\nuse function in_array;\nuse function is_numeric;\n\n/** @extends AbstractLexer<TokenType, string|int> */\nclass EnumLexer extends AbstractLexer\n{\n    /**\n     * {@inheritDoc}\n     */\n    protected function getCatchablePatterns(): array\n    {\n        return [\n            '=|<|>',\n            '[a-z]+',\n            '\\d+',\n        ];\n    }\n\n    /**\n     * {@inheritDoc}\n     */\n    protected function getNonCatchablePatterns(): array\n    {\n        return [\n            '\\s+',\n            '(.)',\n        ];\n    }\n\n    protected function getType(string &$value): TokenType\n    {\n        if (is_numeric($value)) {\n            $value = (int) $value;\n\n            return TokenType::INT;\n        }\n\n        if (in_array($value, ['=', '<', '>'])) {\n            return TokenType::OPERATOR;\n        }\n\n        return TokenType::STRING;\n    }\n}\n"
  },
  {
    "path": "tests/MutableLexer.php",
    "content": "<?php\n\ndeclare(strict_types=1);\n\nnamespace Doctrine\\Tests\\Common\\Lexer;\n\nuse Doctrine\\Common\\Lexer\\AbstractLexer;\n\n/** @extends AbstractLexer<int, string> */\nclass MutableLexer extends AbstractLexer\n{\n    /** @var string[] */\n    private array $catchablePatterns = [];\n\n    public function addCatchablePattern(string $pattern): void\n    {\n        $this->catchablePatterns[] = $pattern;\n    }\n\n    /**\n     * {@inheritDoc}\n     */\n    protected function getCatchablePatterns(): array\n    {\n        return $this->catchablePatterns;\n    }\n\n    /**\n     * {@inheritDoc}\n     */\n    protected function getNonCatchablePatterns(): array\n    {\n        return ['[\\s,]+'];\n    }\n\n    protected function getType(string &$value): int\n    {\n        return 1;\n    }\n}\n"
  },
  {
    "path": "tests/TokenTest.php",
    "content": "<?php\n\ndeclare(strict_types=1);\n\nnamespace Doctrine\\Tests\\Common\\Lexer;\n\nuse Doctrine\\Common\\Lexer\\Token;\nuse PHPUnit\\Framework\\TestCase;\n\nfinal class TokenTest extends TestCase\n{\n    public function testIsA(): void\n    {\n        /** @var Token<'string'|'int', string> $token */\n        $token = new Token('foo', 'string', 1);\n\n        self::assertTrue($token->isA('string'));\n        self::assertTrue($token->isA('int', 'string'));\n        self::assertFalse($token->isA('int'));\n    }\n}\n"
  },
  {
    "path": "tests/TokenType.php",
    "content": "<?php\n\ndeclare(strict_types=1);\n\nnamespace Doctrine\\Tests\\Common\\Lexer;\n\nenum TokenType\n{\n    case INT;\n    case OPERATOR;\n    case STRING;\n}\n"
  }
]