[ { "path": ".editorconfig", "content": "root = true\n\n[*]\nend_of_line = lf\ncharset = utf-8\ntrim_trailing_whitespace = true\ninsert_final_newline = true\nindent_style = tab\nindent_size = 4\n\n[*.md]\ntrim_trailing_whitespace = false\n" }, { "path": ".gitignore", "content": "target\nCargo.lock\n" }, { "path": ".travis.yml", "content": "language: rust\nsudo: false\n\n# run builds for all the trains (and more)\nrust:\n - stable\n\n# load travis-cargo\nbefore_script:\n - |\n pip install 'travis-cargo<0.2' --user &&\n export PATH=$HOME/.local/bin:$PATH\n# the main build\nscript:\n - |\n travis-cargo build &&\n travis-cargo test\n\nenv:\n global:\n # override the default `--features unstable` used for the nightly branch (optional)\n - TRAVIS_CARGO_NIGHTLY_FEATURE=\"\"\n" }, { "path": ".vscode/launch.json", "content": "{\n \"version\": \"0.2.0\",\n \"configurations\": [\n\t {\n\t\t \"type\": \"lldb\",\n\t\t \"request\": \"launch\",\n\t\t \"name\": \"Custom launch\",\n\t\t \"program\": \"./target/debug/examples/json\"\n\t },\n\t {\n\t\t \"type\": \"lldb-mi\",\n\t\t \"request\": \"launch\",\n\t\t \"name\": \"Launch Program\",\n\t\t \"target\": \"./target/debug/examples/simple\",\n\t\t \"cwd\": \"${workspaceRoot}\"\n\t }\n ]\n}\n" }, { "path": ".vscode/tasks.json", "content": "{\n\t// See https://go.microsoft.com/fwlink/?LinkId=733558\n\t// for the documentation about the tasks.json format\n\t\"version\": \"0.1.0\",\n\t\"command\": \"cargo\",\n\t\"isShellCommand\": true,\n\t\"showOutput\": \"always\",\n\t\"echoCommand\": true,\n\t\"suppressTaskName\": true,\n\t\"tasks\": [\n\t\t{\n\t\t\t\"taskName\": \"json\",\n\t\t\t\"args\": [\n\t\t\t\t\"build\",\n\t\t\t\t\"--example\",\n\t\t\t\t\"json\"\n\t\t\t]\n\t\t},\n\t\t{\n\t\t\t\"taskName\": \"release\",\n\t\t\t\"args\": [\n\t\t\t\t\"build\",\n\t\t\t\t\"--release\"\n\t\t\t]\n\t\t}\n\t]\n}\n" }, { "path": "Cargo.toml", "content": "[package]\nname = \"pom\"\nversion = \"3.4.1\"\nedition = \"2021\"\nauthors = [\"Junfeng Liu \"]\nhomepage = \"https://github.com/J-F-Liu/pom\"\ndocumentation = \"https://docs.rs/crate/pom/\"\nrepository = \"https://github.com/J-F-Liu/pom.git\"\nlicense = \"MIT\"\nreadme = \"README.md\"\ndescription = \"PEG parser combinators using operator overloading without macros.\"\ncategories = [\"parsing\"]\nkeywords = [\"parser\", \"parser-combinators\", \"parsing\", \"PEG\"]\n\n[badges]\ntravis-ci = { repository = \"J-F-Liu/pom\" }\n\n[dependencies]\nbstr = { version = \"1.1.0\", features = [\n] } # Only uses one function, so no features needed.\n\n[features]\ndefault = [\"utf8\"]\nutf8 = []\ntrace = []\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2016 Junfeng Liu\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "# pom\n\n[![Crates.io](https://img.shields.io/crates/v/pom.svg)](https://crates.io/crates/pom)\n[![Build Status](https://travis-ci.org/J-F-Liu/pom.png)](https://travis-ci.org/J-F-Liu/pom)\n[![Docs](https://docs.rs/pom/badge.svg)](https://docs.rs/pom)\n[![Discord](https://img.shields.io/badge/discord-pom-red.svg)](https://discord.gg/CVy85pg)\n\nPEG parser combinators created using operator overloading without macros.\n\n## Document\n\n- [Tutorial](https://github.com/J-F-Liu/pom/blob/master/doc/article.md)\n- [API Reference](https://docs.rs/crate/pom/)\n- [Learning Parser Combinators With Rust](https://bodil.lol/parser-combinators/) - By Bodil Stokke\n\n## What is PEG?\n\nPEG stands for parsing expression grammar, is a type of analytic formal grammar, i.e. it describes a formal language in terms of a set of rules for recognizing strings in the language.\nUnlike CFGs, PEGs cannot be ambiguous; if a string parses, it has exactly one valid parse tree.\nEach parsing function conceptually takes an input string as its argument, and yields one of the following results:\n- success, in which the function may optionally move forward or consume one or more characters of the input string supplied to it, or\n- failure, in which case no input is consumed.\n\nRead more on [Wikipedia](https://en.wikipedia.org/wiki/Parsing_expression_grammar).\n\n## What is parser combinator?\n\nA parser combinator is a higher-order function that accepts several parsers as input and returns a new parser as its output.\nParser combinators enable a recursive descent parsing strategy that facilitates modular piecewise construction and testing.\n\nParsers built using combinators are straightforward to construct, readable, modular, well-structured and easily maintainable.\nWith operator overloading, a parser combinator can take the form of an infix operator, used to glue different parsers to form a complete rule.\nParser combinators thereby enable parsers to be defined in an embedded style, in code which is similar in structure to the rules of the formal grammar.\nAnd the code is easier to debug than macros.\n\nThe main advantage is that you don't need to go through any kind of code generation step, you're always using the vanilla language underneath.\nAside from build issues (and the usual issues around error messages and debuggability, which in fairness are about as bad with macros as with code generation), it's usually easier to freely intermix grammar expressions and plain code.\n\n## List of predefined parsers and combinators\n\n| Basic Parsers | Description |\n|------------------|-----------------------------------------------------------------|\n| empty() | Always succeeds, consume no input. |\n| end() | Match end of input. |\n| any() | Match any symbol and return the symbol. |\n| sym(t) | Match a single terminal symbol _t_. |\n| seq(s) | Match sequence of symbols. |\n| list(p,s) | Match list of _p_, separated by _s_. |\n| one_of(set) | Success when current input symbol is one of the set. |\n| none_of(set) | Success when current input symbol is none of the set. |\n| is_a(predicate) | Success when predicate return true on current input symbol. |\n| not_a(predicate) | Success when predicate return false on current input symbol. |\n| take(n) | Read _n_ symbols. |\n| skip(n) | Skip _n_ symbols. |\n| call(pf) | Call a parser factory, can be used to create recursive parsers. |\n\n| Parser Combinators | Description |\n|--------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| p | q | Match p or q, return result of the first success. |\n| p + q | Match p and q, if both succeed return a pair of results. |\n| p - q | Match p and q, if both succeed return result of p. |\n| p \\* q | Match p and q, if both succeed return result of q. |\n| p >> q | Parse p and get result P, then parse q and return result of q(P). |\n| -p | Success when p succeeds, doesn't consume input. |\n| !p | Success when p fails, doesn't consume input. |\n| p.opt() | Make parser optional. Returns an `Option`. |\n| p.repeat(m..n) | `p.repeat(0..)` repeat p zero or more times
`p.repeat(1..)` repeat p one or more times
`p.repeat(1..4)` match p at least 1 and at most 3 times
`p.repeat(5)` repeat p exactly 5 times |\n| p.map(f) | Convert parser result to desired value. |\n| p.convert(f) | Convert parser result to desired value, fails in case of conversion error. |\n| p.pos() | Get input position after matching p. |\n| p.collect() | Collect all matched input symbols. |\n| p.discard() | Discard parser output. |\n| p.name(\\_) | Give parser a name to identify parsing errors.
If the `trace` feature is enabled then a basic trace for the parse and parse result is made to stderr. |\n| p.expect(\\_) | Mark parser as expected, abort early when failed in ordered choice. |\n\nThe choice of operators is established by their operator precedence, arity and \"meaning\".\nUse `*` to ignore the result of first operand on the start of an expression, `+` and `-` can fulfill the need on the rest of the expression.\n\nFor example, `A * B * C - D + E - F` will return the results of C and E as a pair.\n\n## Example code\n\n```rust\nuse pom::parser::*;\n\nlet input = b\"abcde\";\nlet parser = sym(b'a') * none_of(b\"AB\") - sym(b'c') + seq(b\"de\");\nlet output = parser.parse(input);\nassert_eq!(output, Ok( (b'b', vec![b'd', b'e'].as_slice()) ) );\n```\n\n### Example JSON parser\n\n```rust\nextern crate pom;\nuse pom::parser::*;\nuse pom::Parser;\n\nuse std::collections::HashMap;\nuse std::str::{self, FromStr};\n\n#[derive(Debug, PartialEq)]\npub enum JsonValue {\n\tNull,\n\tBool(bool),\n\tStr(String),\n\tNum(f64),\n\tArray(Vec),\n\tObject(HashMap)\n}\n\nfn space() -> Parser {\n\tone_of(b\" \\t\\r\\n\").repeat(0..).discard()\n}\n\nfn number() -> Parser {\n\tlet integer = one_of(b\"123456789\") - one_of(b\"0123456789\").repeat(0..) | sym(b'0');\n\tlet frac = sym(b'.') + one_of(b\"0123456789\").repeat(1..);\n\tlet exp = one_of(b\"eE\") + one_of(b\"+-\").opt() + one_of(b\"0123456789\").repeat(1..);\n\tlet number = sym(b'-').opt() + integer + frac.opt() + exp.opt();\n\tnumber.collect().convert(str::from_utf8).convert(|s|f64::from_str(&s))\n}\n\nfn string() -> Parser {\n\tlet special_char = sym(b'\\\\') | sym(b'/') | sym(b'\"')\n\t\t| sym(b'b').map(|_|b'\\x08') | sym(b'f').map(|_|b'\\x0C')\n\t\t| sym(b'n').map(|_|b'\\n') | sym(b'r').map(|_|b'\\r') | sym(b't').map(|_|b'\\t');\n\tlet escape_sequence = sym(b'\\\\') * special_char;\n\tlet string = sym(b'\"') * (none_of(b\"\\\\\\\"\") | escape_sequence).repeat(0..) - sym(b'\"');\n\tstring.convert(String::from_utf8)\n}\n\nfn array() -> Parser> {\n\tlet elems = list(call(value), sym(b',') * space());\n\tsym(b'[') * space() * elems - sym(b']')\n}\n\nfn object() -> Parser> {\n\tlet member = string() - space() - sym(b':') - space() + call(value);\n\tlet members = list(member, sym(b',') * space());\n\tlet obj = sym(b'{') * space() * members - sym(b'}');\n\tobj.map(|members|members.into_iter().collect::>())\n}\n\nfn value() -> Parser {\n\t( seq(b\"null\").map(|_|JsonValue::Null)\n\t| seq(b\"true\").map(|_|JsonValue::Bool(true))\n\t| seq(b\"false\").map(|_|JsonValue::Bool(false))\n\t| number().map(|num|JsonValue::Num(num))\n\t| string().map(|text|JsonValue::Str(text))\n\t| array().map(|arr|JsonValue::Array(arr))\n\t| object().map(|obj|JsonValue::Object(obj))\n\t) - space()\n}\n\npub fn json() -> Parser {\n\tspace() * value() - end()\n}\n\nfn main() {\n\tlet input = br#\"\n\t{\n \"Image\": {\n \"Width\": 800,\n \"Height\": 600,\n \"Title\": \"View from 15th Floor\",\n \"Thumbnail\": {\n \"Url\": \"http://www.example.com/image/481989943\",\n \"Height\": 125,\n \"Width\": 100\n },\n \"Animated\" : false,\n \"IDs\": [116, 943, 234, 38793]\n }\n }\"#;\n\n\tprintln!(\"{:?}\", json().parse(input));\n}\n```\n\nYou can run this example with the following command:\n\n```\ncargo run --example json\n```\n\n## Benchmark\n\n| Parser | Time to parse the same JSON file |\n|------------------------------------------------------|----------------------------------|\n| pom: json_byte | 621,319 ns/iter (+/- 20,318) |\n| pom: json_char | 627,110 ns/iter (+/- 11,463) |\n| [pest](https://github.com/dragostis/pest): json_char | 13,359 ns/iter (+/- 811) |\n\n### Lifetimes and files\n\nString literals have a static lifetime so they can work with the static version of Parser\nimported from `pom::Parser`. Input read from a file has a shorter lifetime. In this case you\nshould import `pom::parser::Parser` and declare lifetimes on your parser functions. So\n\n```rust\nfn space() -> Parser {\n one_of(b\" \\t\\r\\n\").repeat(0..).discard()\n}\n\n```\n\nwould become\n\n```rust\nfn space<'a>() -> Parser<'a, u8, ()> {\n one_of(b\" \\t\\r\\n\").repeat(0..).discard()\n}\n```\n" }, { "path": "assets/data.json", "content": "[\n {\n \"_id\": \"5741cfe6bf9f447a509a269e\",\n \"index\": 0,\n \"guid\": \"642f0c2a-3d87-43ac-8f82-25f004e0c96a\",\n \"isActive\": false,\n \"balance\": \"$3,666.68\",\n \"picture\": \"http://placehold.it/32x32\",\n \"age\": 39,\n \"eyeColor\": \"blue\",\n \"name\": \"Leonor Herman\",\n \"gender\": \"female\",\n \"company\": \"RODEOMAD\",\n \"email\": \"leonorherman@rodeomad.com\",\n \"phone\": \"+1 (848) 456-2962\",\n \"address\": \"450 Seeley Street, Iberia, North Dakota, 7859\",\n \"about\": \"Reprehenderit in anim laboris labore sint occaecat labore proident ipsum exercitation. Ut ea aliqua duis occaecat consectetur aliqua anim id. Dolor ea fugiat excepteur reprehenderit eiusmod enim non sit nisi. Mollit consequat anim mollit et excepteur qui laborum qui eiusmod. Qui ea amet incididunt cillum quis occaecat excepteur qui duis nisi. Dolore labore eu sunt consequat magna.\\r\\n\",\n \"registered\": \"2015-03-06T02:49:06 -02:00\",\n \"latitude\": -29.402032,\n \"longitude\": 151.088135,\n \"tags\": [\n \"Lorem\",\n \"voluptate\",\n \"aute\",\n \"ullamco\",\n \"elit\",\n \"esse\",\n \"culpa\"\n ],\n \"friends\": [\n {\n \"id\": 0,\n \"name\": \"Millicent Norman\"\n },\n {\n \"id\": 1,\n \"name\": \"Vincent Cannon\"\n },\n {\n \"id\": 2,\n \"name\": \"Gray Berry\"\n }\n ],\n \"greeting\": \"Hello, Leonor Herman! You have 4 unread messages.\",\n \"favoriteFruit\": \"apple\"\n },\n {\n \"_id\": \"5741cfe69424f42d4493caa2\",\n \"index\": 1,\n \"guid\": \"40ec6b43-e6e6-44e1-92a8-dc80cd5d7179\",\n \"isActive\": true,\n \"balance\": \"$2,923.78\",\n \"picture\": \"http://placehold.it/32x32\",\n \"age\": 36,\n \"eyeColor\": \"blue\",\n \"name\": \"Barton Barnes\",\n \"gender\": \"male\",\n \"company\": \"BRAINQUIL\",\n \"email\": \"bartonbarnes@brainquil.com\",\n \"phone\": \"+1 (907) 553-3739\",\n \"address\": \"644 Falmouth Street, Sedley, Michigan, 5602\",\n \"about\": \"Et nulla laboris consectetur laborum labore. Officia dolor sint do amet excepteur dolore eiusmod. Occaecat pariatur sunt velit sunt ullamco labore commodo mollit sint dolore occaecat.\\r\\n\",\n \"registered\": \"2014-08-28T01:07:22 -03:00\",\n \"latitude\": 14.056553,\n \"longitude\": -61.911624,\n \"tags\": [\n \"laboris\",\n \"sunt\",\n \"esse\",\n \"tempor\",\n \"pariatur\",\n \"occaecat\",\n \"et\"\n ],\n \"friends\": [\n {\n \"id\": 0,\n \"name\": \"Tillman Mckay\"\n },\n {\n \"id\": 1,\n \"name\": \"Rivera Berg\"\n },\n {\n \"id\": 2,\n \"name\": \"Rosetta Erickson\"\n }\n ],\n \"greeting\": \"Hello, Barton Barnes! You have 2 unread messages.\",\n \"favoriteFruit\": \"banana\"\n }\n]\n" }, { "path": "benches/json.rs", "content": "#![feature(test)]\nextern crate test;\nuse self::test::Bencher;\n\nuse std::fs::File;\nuse std::io::Read;\n\nextern crate pom;\n\n#[path = \"../examples/json.rs\"]\nmod json;\n\n#[bench]\nfn json_byte(b: &mut Bencher) {\n\tlet mut file = File::open(\"assets/data.json\").unwrap();\n\tlet mut input = Vec::new();\n\tfile.read_to_end(&mut input).unwrap();\n\n\tb.iter(|| {\n\t\tjson::json().parse(&input).ok();\n\t});\n}\n" }, { "path": "benches/json_char.rs", "content": "#![feature(test)]\nextern crate test;\nuse self::test::Bencher;\n\nuse std::fs::File;\nuse std::io::Read;\n\nextern crate pom;\n\n#[path = \"../examples/json_char.rs\"]\nmod json;\n\n#[bench]\nfn json_char(b: &mut Bencher) {\n\tlet mut file = File::open(\"assets/data.json\").unwrap();\n\tlet mut input = String::new();\n\tfile.read_to_string(&mut input).unwrap();\n\tlet chars: Vec = input.chars().collect();\n\n\tb.iter(|| {\n\t\tjson::json().parse(&chars).ok();\n\t});\n}\n" }, { "path": "doc/article.md", "content": "# PEG Parser Combinators Implemented in Rust\n\nThis article introduces [pom](https://github.com/J-F-Liu/pom), a PEG parser combinator library implemented in Rust, using operator overloading without macros.\n\n## Why Rust?\n\n![Rust](rust.png)\n\nAfter I've learned C/C++ and C#, I found that choosing a new programming language can greatly affect a programmer's productivity.\nOn one hand I keep sorting out new languages, there are hundreds of them, I examine and choose what I like best, my favorites are C#, Ruby, TypeScript and Rust.\nOn the other hand I try to design a new language and implement a compiler by myself.\n\nI like the syntax provided by C#, but hate the huge .NET runtime. Dependency on CLR makes distribution of an application written in C# very hard. Compiling to native code is always what I longed for a programming language.\nIn year 2003 I thought a compiler can get rid of garbage collector by generating free memory instructions in appropriate locations in the target program.\nBut I didn't go deep into the design of the details of this mechanism, I decided to firstly write a working compiler, then improve the design of the language and implementation of the compiler bit by bit.\n\nThe first stage of compilation is parsing. I tried some parser generators, but not satisfied with the result.\nThen I dig into the parsing theory, followed several books, implemented DFA, NFA, NFA to DFA conversion, LL(1), LR, LALR algorithms,\nthen wrote a parser to parse BNF, EBNF or TBNF grammar file, and generate parser code corresponding to the grammar.\n\nThe syntax/semantics analysis and code generation parts of a compiler are more difficult.\nI even tried to define a intermediate assembly language, at that time I didn't know LLVM. My effort of writing a compiler ceased for years, then Rust was born.\n\nAt first glance, the Rust's syntax is a bit strange, why use `fn` instaed of `def`, why use `let mut` instead of `var`, I was not attracted by it.\nAfter read a publication on O'Reilly [*Why Rust?*](http://www.oreilly.com/programming/free/files/why-rust.pdf) I suddenly realized that this is language I'm trying to build,\nwhen you actually start using Rust you'll find that `fn` and `let mut` fits Rust's logic well. For me, **Rust is once a dream now a reality.**\n\nRust has a steep learning curve, more challenging than any of the previous programming languages I learned. All this learning is worthwhile when you finally get your program working and polished.\nObject oriented class hierarchy is not good enough for code reuse, Rust's enum, tuple, struct and trait type system is a better solution.\nI still wondering whether the Rust compiler can be smart enough to elide all the lifetime parameters, they are mostly noise and obstacle when reading and writing programs.\n\n## What is PEG?\n\nWhen I discovered [PEG](http://bford.info/packrat/), I knew that all my previous work on LALR can be thrown away.\nI rewrote my parser generator using and working with PEG. Using this parser generator I created a [YAML parser](https://www.codeproject.com/Articles/28720/YAML-Parser-in-C) and a [Lua Interpreter](https://www.codeproject.com/Articles/228212/Lua-Interpreter).\n\n[Parsing Expression Grammars](http://en.wikipedia.org/wiki/Parsing_expression_grammar) (PEGs) are an alternative to [Context-Free Grammars](http://en.wikipedia.org/wiki/Context-free_grammar) (CFGs) for formally specifying syntax.\nCFG describe a rule system to generate language strings while PEG describe a rule system to recognize language strings.\n\n![CFG](cfg.png)\n\n![PEG](peg.png)\n\nUnlike CFGs, PEGs cannot be ambiguous; if a string parses, it has exactly one valid parse tree.\nWe normally specify our languages directly by how to recognize it, so PEG is both closer match to syntax practices and more powerful than nondeterministic CFG.\n\n### Parsing expressions\n\n| Expr | Description |\n| ---------- | ----------------------------------- |\n| ε | the empty string |\n| a | terminal (a ∈ Σ) |\n| A | non-terminal (A ∈ N) |\n| e1 e2 | sequence |\n| e1 / e2 | prioritized choice |\n| e?, e*, e+ | optional, zero-or-more, one-or-more |\n| &e, !e | syntactic predicates |\n\n## What is parser combinator?\n\nWhen I heard of Parsec in the Haskell world, I got the concept of parser combinator for my first time.\n\nA *parser* is a function which takes a *string* (a series of *symbols*) as input, and returns matching result as *output*.\n\nA *combinator* is a higher-order function (a \"functional\") which takes zero or more functions (each of the same type) as input and returns a new function of the same type as output.\n\nA *parser combinator* is a higher-order function which takes parsers as input and returns a new parser as output.\n\nParser combinators allow you write grammar rules and create a parser directly in the host language, without a separated parser generation step, so the whole procedure is more fluent.\n\n## How to implement parser combinators?\n\nI thought deeply about how to implement parser combinator using language constructs provided by Rust. In summary, there are four approaches:\n\n1. Parser as closure\n\n ```rust\n pub fn empty() -> impl Fn(&mut Input) -> Result<()> {\n |_: &mut Input| Ok(())\n }\n\n pub fn term(t: I) -> impl Fn(&mut Input) -> Result {\n ...\n }\n\n pub fn seq<'a, I>(tag: &'a [I]) -> impl Fn(&mut Input) -> Result<&'a [I]> {\n ...\n }\n ...\n // To create a parser for integer\n let parser = concatenate(optional(one_of(\"+-\")), one_or_more(one_of(\"0123456789\")));\n ```\n\n *Pros*: Less implementation code.\n\n *Cons*: Cannot overload operators, poor readability.\n\n2. Parser as struct\n\n ```rust\n pub struct Parser {\n method: Box) -> Result>,\n }\n\n impl Parser {\n /// Create new parser.\n pub fn new

(parse: P) -> Parser\n where P: Fn(&mut Input) -> Result + 'static\n {\n Parser { method: Box::new(parse) }\n }\n\n /// Apply the parser to parse input.\n pub fn parse(&self, input: &mut Input) -> Result {\n (self.method)(input)\n }\n ...\n }\n\n pub fn empty() -> Parser {\n Parser::new(|_: &mut Input| Ok(()))\n }\n\n pub fn term(t: I) -> Parser {\n ...\n }\n ...\n impl Add> for Parser {\n type Output = Parser;\n\n fn add(self, other: Parser) -> Self::Output\n where I: 'static,\n O: 'static,\n U: 'static\n {\n Parser::new(move |input: &mut Input| {\n let start = input.position();\n let result = self.parse(input)\n .and_then(|out1| other.parse(input).map(|out2| (out1, out2)));\n if result.is_err() {\n input.jump_to(start);\n }\n result\n })\n }\n }\n ...\n // To create a parser for integer\n let parser = one_of(\"+-\").opt() + one_of(\"0123456789\").repeat(1..);\n ```\n\n *Pros*: Can overload operators, elegant code.\n\n *Cons*: Depends on compiler's zero-cost abstractions to optimize runtime performance.\n\n Crate [pom](https://github.com/J-F-Liu/pom) is using this approach.\n\n3. Parser as trait\n\n ```rust\n pub trait Parser {\n type I: ?Sized;\n type O;\n\n fn parse<'a>(&self, data: &'a Self::I) -> ParseResult<&'a Self::I, Self::O>;\n }\n\n pub trait ParserCombinator : Parser + Clone {\n fn then>(&self, p: P) -> ChainedParser {\n ChainedParser{first: self.clone(), second: p}\n }\n ...\n }\n\n pub fn opt(t: T) -> OptionParser {\n OptionParser{parser: t}\n }\n\n pub fn recursive Box>>(f: F) -> RecursiveParser {\n RecursiveParser{parser: Rc::new(f)}\n }\n\n ...\n\n pub struct ChainedParser {\n first: A,\n second: B,\n }\n ...\n impl, B: Parser> Parser for ChainedParser {\n type I = C;\n type O = (A::O,B::O);\n\n fn parse<'a>(&self, data: &'a Self::I) -> ParseResult<&'a Self::I, Self::O>{\n match self.first.parse(data) {\n Ok((a, d2)) => match self.second.parse(d2) {\n Ok((b, remain)) => Ok(((a, b), remain)),\n Err(err) => Err(err)\n },\n Err(err) => Err(err)\n }\n }\n }\n\n impl, B: ParserCombinator> Clone for ChainedParser {\n ...\n }\n ...\n ```\n\n *Pros*: Can overload operators.\n\n *Cons*: Bloated code.\n\n Crate [peruse](https://github.com/DanSimon/peruse) is using this approach.\n\n4. Parser as macro\n\n ```rust\n #[macro_export]\n macro_rules! do_parse (\n (__impl $i:expr, $consumed:expr, ( $($rest:expr),* )) => (\n $crate::IResult::Done($i, ( $($rest),* ))\n );\n\n (__impl $i:expr, $consumed:expr, $e:ident >> $($rest:tt)*) => (\n do_parse!(__impl $i, $consumed, call!($e) >> $($rest)*);\n );\n (__impl $i:expr, $consumed:expr, $submac:ident!( $($args:tt)* ) >> $($rest:tt)*) => (\n {\n match $submac!($i, $($args)*) {\n $crate::IResult::Error(e) => $crate::IResult::Error(e),\n $crate::IResult::Incomplete($crate::Needed::Unknown) =>\n $crate::IResult::Incomplete($crate::Needed::Unknown),\n $crate::IResult::Incomplete($crate::Needed::Size(i)) =>\n $crate::IResult::Incomplete($crate::Needed::Size($consumed + i)),\n $crate::IResult::Done(i,_) => {\n do_parse!(__impl i,\n $consumed + ($crate::InputLength::input_len(&($i)) -\n $crate::InputLength::input_len(&i)), $($rest)*)\n },\n }\n }\n );\n\n (__impl $i:expr, $consumed:expr, $field:ident : $e:ident >> $($rest:tt)*) => (\n do_parse!(__impl $i, $consumed, $field: call!($e) >> $($rest)*);\n );\n\n (__impl $i:expr, $consumed:expr, $field:ident : $submac:ident!( $($args:tt)* ) >> $($rest:tt)*) => (\n {\n match $submac!($i, $($args)*) {\n $crate::IResult::Error(e) => $crate::IResult::Error(e),\n $crate::IResult::Incomplete($crate::Needed::Unknown) =>\n $crate::IResult::Incomplete($crate::Needed::Unknown),\n $crate::IResult::Incomplete($crate::Needed::Size(i)) =>\n $crate::IResult::Incomplete($crate::Needed::Size($consumed + i)),\n $crate::IResult::Done(i,o) => {\n let $field = o;\n do_parse!(__impl i,\n $consumed + ($crate::InputLength::input_len(&($i)) -\n $crate::InputLength::input_len(&i)), $($rest)*)\n },\n }\n }\n );\n\n // ending the chain\n (__impl $i:expr, $consumed:expr, $e:ident >> ( $($rest:tt)* )) => (\n do_parse!(__impl $i, $consumed, call!($e) >> ( $($rest)* ));\n );\n\n (__impl $i:expr, $consumed:expr, $submac:ident!( $($args:tt)* ) >> ( $($rest:tt)* )) => (\n match $submac!($i, $($args)*) {\n $crate::IResult::Error(e) => $crate::IResult::Error(e),\n $crate::IResult::Incomplete($crate::Needed::Unknown) =>\n $crate::IResult::Incomplete($crate::Needed::Unknown),\n $crate::IResult::Incomplete($crate::Needed::Size(i)) =>\n $crate::IResult::Incomplete($crate::Needed::Size($consumed + i)),\n $crate::IResult::Done(i,_) => {\n $crate::IResult::Done(i, ( $($rest)* ))\n },\n }\n );\n\n (__impl $i:expr, $consumed:expr, $field:ident : $e:ident >> ( $($rest:tt)* )) => (\n do_parse!(__impl $i, $consumed, $field: call!($e) >> ( $($rest)* ) );\n );\n\n (__impl $i:expr, $consumed:expr, $field:ident : $submac:ident!( $($args:tt)* ) >> ( $($rest:tt)* )) => (\n match $submac!($i, $($args)*) {\n $crate::IResult::Error(e) => $crate::IResult::Error(e),\n $crate::IResult::Incomplete($crate::Needed::Unknown) =>\n $crate::IResult::Incomplete($crate::Needed::Unknown),\n $crate::IResult::Incomplete($crate::Needed::Size(i)) =>\n $crate::IResult::Incomplete($crate::Needed::Size($consumed + i)),\n $crate::IResult::Done(i,o) => {\n let $field = o;\n $crate::IResult::Done(i, ( $($rest)* ))\n },\n }\n );\n\n ($i:expr, $($rest:tt)*) => (\n {\n do_parse!(__impl $i, 0usize, $($rest)*)\n }\n );\n );\n ...\n // To create a parser for integer\n named!(integer<&[u8], i64>, map!(\n pair!(\n opt!(sign),\n map_res!(map_res!(digit, str::from_utf8), i64::from_str)\n ),\n |(sign, value): (Option, i64)| { sign.unwrap_or(1) * value }\n ));\n ```\n\n *Pros*: Can create DSL syntax, high performance.\n\n *Cons*: Macros themselves are difficult to read, write and debug.\n\n\nAccording to above comparison, parser as struct is the best approach. At first I choose to use nom to create a PDF parser, it turns out a special PDF feature blocked me.\nWhen parsing a PDF stream object, it's length may be a referenced object, hence the need to get the length from a reader.\nThe `named! ` macro cannot accept extra parameters, there is no obvious way to read a length object inside a stream object parser.\nThis is the primary reason why I started to develop pom.\n\n## List of predefined parsers and combinators in pom\n\n| Basic Parsers | Description |\n| -------------- | ---------------------------------------- |\n| empty() | Always succeeds, consume no input. |\n| end() | Match end of input. |\n| sym(t) | Match a single terminal symbol *t*. |\n| seq(s) | Match sequence of symbols. |\n| list(p,s) | Match list of *p*, separated by *s*. |\n| one_of(set) | Success when current input symbol is one of the set. |\n| none_of(set) | Success when current input symbol is none of the set. |\n| is_a(predicate) | Success when predicate return true on current input symbol. |\n| not_a(predicate) | Success when predicate return false on current input symbol. |\n| take(n) | Read *n* symbols. |\n| skip(n) | Skip *n* symbols. |\n| call(pf) | Call a parser factory, can used to create recursive parsers. |\n\nThese are functions to create basic parsers.\n\n\n| Parser Combinators | Description |\n| ------------------ | ---------------------------------------- |\n| p | q | Match p or q, return result of the first success. |\n| p + q | Match p and q, if both success return a pair of results. |\n| p - q | Match p and q, if both success return result of p. |\n| p * q | Match p and q, if both success return result of q. |\n| p >> q | Parse p and get result P, then parse and return result of q(P). |\n| -p | Success when p success, doen't consume input. |\n| !p | Success when p fail, doen't consume input. |\n| p.opt() | Make parser optional. |\n| p.repeat(m..n) | `p.repeat(0..)` repeat p zero or more times
`p.repeat(1..)` repeat p one or more times
`p.repeat(1..4)` match p at least 1 and at most 3 times
`p.repeat(1..=3)` also match p at least 1 and at most 3 times |\n| p.map(f) | Convert parser result to desired value. |\n| p.convert(f) | Convert parser result to desired value, fail in case of conversion error. |\n| p.pos() | Get input position after matching p. |\n| p.collect() | Collect all matched input symbols. |\n| p.discard() | Discard parser output. |\n| p.name(_) | Give parser a name to identify parsing errors. |\n\nThese are operations to create new parsers based on other parsers. The choice of operators is established by their operator precedence, arity and \"meaning\".\n\nUse `*` to ignore the result of first operand on the start of an expression, `+` and `-` can fulfill the need on the rest of the expression.\nFor example, `A * B * C - D + E - F` will return the results of C and E as a pair.\n\n## Using the code\n\nThere are three ways to create a parser:\n\n1. As a variable, normally used to construct another parser.\n\n ```rust\n let integer = one_of(b\"+-\").opt() + one_of(b\"0123456789\").repeat(1..);\n ```\n\n2. As a closure, when referenced several times in constructing another parser.\n\n ```rust\n let integer = || one_of(b\"+-\").opt() + one_of(b\"0123456789\").repeat(1..);\n let pair = sym(b'(') * integer() - sym(b',') + integer() - sym(b')');\n ```\n\n3. As a function, provides a high level construct.\n\n ```rust\n fn integer() -> Parser, Vec)> {\n one_of(b\"+-\").opt() + one_of(b\"0123456789\").repeat(1..)\n }\n ```\n\n## Example JSON Parser\n\nLet me explain the parser combinators in more detail by creating a JSON parser. Syntax diagrams can be found on [json.org](http://www.json.org/).\n\n```rust\nextern crate pom;\nuse pom::{Parser, DataInput};\nuse pom::char_class::hex_digit;\nuse pom::parser::*;\n\nuse std::str::FromStr;\nuse std::char::{decode_utf16, REPLACEMENT_CHARACTER};\nuse std::collections::HashMap;\n\n#[derive(Debug, PartialEq)]\npub enum JsonValue {\n Null,\n Bool(bool),\n Str(String),\n Num(f64),\n Array(Vec),\n Object(HashMap)\n}\n```\n\nImport predefined parser combinators and utility functions, define the JSON parser's output value as an enum.\n\n```rust\nfn space() -> Parser {\n one_of(b\" \\t\\r\\n\").repeat(0..).discard()\n}\n```\n\nMatch zero or more space characters, the output is ignored.\n\n```rust\nfn number() -> Parser {\n let integer = one_of(b\"123456789\") - one_of(b\"0123456789\").repeat(0..) | sym(b'0');\n let frac = sym(b'.') + one_of(b\"0123456789\").repeat(1..);\n let exp = one_of(b\"eE\") + one_of(b\"+-\").opt() + one_of(b\"0123456789\").repeat(1..);\n let number = sym(b'-').opt() + integer + frac.opt() + exp.opt();\n number.collect().convert(String::from_utf8).convert(|s|f64::from_str(&s))\n}\n```\n\nDon't care each output of integer, frac or exp, collect() method get all the match character as a Vec, then it is converted to a string, and further converted to a float number.\n\n```rust\nfn string() -> Parser {\n let special_char = sym(b'\\\\') | sym(b'/') | sym(b'\"')\n | sym(b'b').map(|_|b'\\x08') | sym(b'f').map(|_|b'\\x0C')\n | sym(b'n').map(|_|b'\\n') | sym(b'r').map(|_|b'\\r') | sym(b't').map(|_|b'\\t');\n let escape_sequence = sym(b'\\\\') * special_char;\n let char_string = (none_of(b\"\\\\\\\"\") | escape_sequence).repeat(1..).convert(String::from_utf8);\n let utf16_char = seq(b\"\\\\u\") * is_a(hex_digit).repeat(4).convert(String::from_utf8).convert(|digits|u16::from_str_radix(&digits, 16));\n let utf16_string = utf16_char.repeat(1..).map(|chars|decode_utf16(chars).map(|r| r.unwrap_or(REPLACEMENT_CHARACTER)).collect::());\n let string = sym(b'\"') * (char_string | utf16_string).repeat(0..) - sym(b'\"');\n string.map(|strings|strings.concat())\n}\n```\n\nThe bulk of code is written to parse escape sequences.\nAccording to [Wikipedia](https://en.wikipedia.org/wiki/JSON#Data_portability_issues), UTF-16 surrogate pairs is a detail missed by some JSON parsers.\nWe implement this easily with Rust's Unicode support.\n\n```rust\nfn array() -> Parser> {\n let elems = list(call(value), sym(b',') * space());\n sym(b'[') * space() * elems - sym(b']')\n}\n\nfn object() -> Parser> {\n let member = string() - space() - sym(b':') - space() + call(value);\n let members = list(member, sym(b',') * space());\n let obj = sym(b'{') * space() * members - sym(b'}');\n obj.map(|members|members.into_iter().collect::>())\n}\n\nfn value() -> Parser {\n ( seq(b\"null\").map(|_|JsonValue::Null)\n | seq(b\"true\").map(|_|JsonValue::Bool(true))\n | seq(b\"false\").map(|_|JsonValue::Bool(false))\n | number().map(|num|JsonValue::Num(num))\n | string().map(|text|JsonValue::Str(text))\n | array().map(|arr|JsonValue::Array(arr))\n | object().map(|obj|JsonValue::Object(obj))\n ) - space()\n}\n```\n\narray and object are very straight to parse, notice `call(value)`, at the first attempt I write it as `value()`, then an infinite loop is created. Recursive parsing is solved by adding `call()` to `pom`.\n\n```rust\npub fn json() -> Parser {\n space() * value() - end()\n}\n```\n\nThe final JSON parser, declared as public. According to [RFC 7159](https://tools.ietf.org/html/rfc7159) a JSON text is a serialized value of any of the six types.\n`end()` is used to ensure there is no extra text in the input.\n\n```rust\nfn main() {\n let test = br#\"\n {\n \"Image\": {\n \"Width\": 800,\n \"Height\": 600,\n \"Title\": \"View from 15th Floor\",\n \"Thumbnail\": {\n \"Url\": \"http://www.example.com/image/481989943\",\n \"Height\": 125,\n \"Width\": 100\n },\n \"Animated\" : false,\n \"IDs\": [116, 943, 234, 38793]\n },\n \"escaped characters\": \"\\u2192\\uD83D\\uDE00\\\"\\t\\uD834\\uDD1E\"\n }\"#;\n\n let mut input = DataInput::new(test);\n println!(\"{:?}\", json().parse(&mut input));\n}\n```\n\nUse the JSON parser to parse JSON text, the output is:\n\n```\ncargo run --example json\n Compiling pom v0.6.0 (file:///work/pom)\n Finished debug [unoptimized + debuginfo] target(s) in 2.20 secs\n Running `target/debug/examples/json`\nOk(Object({\"Image\": Object({\"Width\": Num(800), \"Title\": Str(\"View from 15th Floor\"), \"Height\": Num(600), \"Animated\": Bool(false), \"IDs\": Array([Num(116), Num(943), Num(234), Num(38793)]), \"Thumbnail\": Object({\"Height\n\": Num(125), \"Url\": Str(\"http://www.example.com/image/481989943\"), \"Width\": Num(100)})}), \"escaped characters\": Str(\"→😀\\\"\\t𝄞\")}))\n```\n\nThe above parser assumes that the input bytes is UTF-8 encoded text; otherwise, you can use the [char version of JSON parser](https://github.com/J-F-Liu/pom/blob/master/examples/json_char.rs).\n\n`p >> q` is not covered in the JSON example. It is used to pass the output of `p` into parser creation of `p`.\n\n```rust\nlet mut input = DataInput::new(b\"5oooooooo\");\nlet parser = one_of(b\"0123456789\").map(|c|c - b'0') >> |n| {\n take(n as usize) + sym(b'o').repeat(0..)\n};\nlet output = parser.parse(&mut input);\nassert_eq!(output, Ok( (vec![b'o';5], vec![b'o';3]) ));\n```\n\nThe first character indicates the number of `o`s to parse, then the number is used in the closure `|n| take(n)`.\n\n## More examples\n\n- A [simple PDF parser](https://github.com/J-F-Liu/lopdf/blob/491dece5867a2b81878208bcb5e07ff1007c0d89/src/parser.rs), you can compare it with the equivalent [nom version](https://github.com/J-F-Liu/lopdf/blob/dff82c49fea9ac9ea23edf42ad80e480bd5edb46/src/parser.rs).\n- A [complete PDF parser](https://github.com/J-F-Liu/lopdf/blob/master/src/parser.rs) which can read length object when parsing stream object.\n\n## Conclusion\n\nI think I created something really cool, you can use pom to write all kinds of parsers elegantly.\nI helped pom to evolve version by version into what it is, and pom also helps me to grow my Rust programming skills a lot.\nOf course there is still room for improvement, any feed back is welcome.\n\n## Points of interest\n\nI try to add a `cache()` method to `Parser`. Memorize the result on given input position, return the result directly when called again, effectively implementing the Packrat Parsing algorithm.\nBut there are two problems, 1) save result means mutate a Hashmap, so Parser's method field should be a Box of `FnMut`,\n2) Hashmap returns an reference of value for a given key, the value cannot be moved, so need to make the value cloneable.\n\n## Pain points where Rust needs to improve\n\n1. Implement trait for `[T]` should automatically implement `[T; N]`.\n2. The standard library should provide a char_at() method return the char and the number of bytes consumed, like:\n\t```rust\n\tpub trait Encoding {\n\t\t/// Get char at a byte index, return the char and the number of bytes read.\n\t\tfn char_at(&self, data: &[u8], index: usize) -> Result<(char, u32)>;\n\t}\n\t```\n3. Can ellide 'static lifetime parameter, allow `Parser<'static, I, O>` written as `Parser`.\n4. Should `impl Copy for closure`, so that FnOnce closure can be passed to map() inside Fn closure.\n\t```rust\n\tpub fn map(self, f: F) -> Parser<'a, I, U>\n\t\twhere F: FnOnce(O) -> U + Copy + 'a,\n\t\t\t I: 'static,\n\t\t\t O: 'static,\n\t\t\t U: 'static\n\t{\n\t\tParser::new(move |input: &mut Input| {\n\t\t\tself.parse(input).map(f)\n\t\t})\n\t}\n\t```\n\n## More Readings\n\n- [The Rust programming language, in the words of its practitioners](https://brson.github.io/fireflowers/)\n- [PEGs, Packrats and Parser Combinators](http://scg.unibe.ch/download/lectures/cc2011/10PEGs.pptx.pdf)\n- [An introduction to parsing text in Haskell with Parsec](http://unbui.lt/#!/post/haskell-parsec-basics/)\n" }, { "path": "examples/duration.rs", "content": "use pom::parser::*;\nuse pom::Parser;\n\nuse std::str::{self, FromStr};\n\n#[derive(Debug, PartialEq)]\nstruct Duration {\n\tyears: Option,\n\tmonths: Option,\n\tweeks: Option,\n\tdays: Option,\n\thours: Option,\n\tminutes: Option,\n\tseconds: Option,\n}\n\nfn number_separator() -> Parser {\n\t// either '.' or ',' can be used as a separator between the whole and decimal part of a number\n\tone_of(b\".,\").discard()\n}\n\nfn number() -> Parser {\n\tlet integer = one_of(b\"0123456789\").repeat(0..);\n\tlet frac = number_separator() + one_of(b\"0123456789\").repeat(1..);\n\tlet number = integer + frac.opt();\n\tnumber\n\t\t.collect()\n\t\t.convert(str::from_utf8)\n\t\t.convert(f32::from_str)\n}\n\nfn date_part() -> Parser, Option, Option, Option)> {\n\t((number() - sym(b'Y')).opt()\n\t\t+ (number() - sym(b'M')).opt()\n\t\t+ (number() - sym(b'W')).opt()\n\t\t+ (number() - sym(b'D')).opt())\n\t.map(|(((years, months), weeks), days)| (years, months, weeks, days))\n}\n\nfn time_part() -> Parser, Option, Option)> {\n\tsym(b'T')\n\t\t* ((number() - sym(b'H')).opt()\n\t\t\t+ (number() - sym(b'M')).opt()\n\t\t\t+ (number() - sym(b'S')).opt())\n\t\t.map(|((hours, minutes), seconds)| (hours, minutes, seconds))\n}\n\nfn parser() -> Parser {\n\tsym(b'P')\n\t\t* (time_part().map(|(hours, minutes, seconds)| Duration {\n\t\t\tyears: None,\n\t\t\tmonths: None,\n\t\t\tweeks: None,\n\t\t\tdays: None,\n\t\t\thours,\n\t\t\tminutes,\n\t\t\tseconds,\n\t\t}) | (date_part() + time_part()).map(|(date_elements, time_elements)| {\n\t\t\tlet (years, months, weeks, days) = date_elements;\n\t\t\tlet (hours, minutes, seconds) = time_elements;\n\t\t\tDuration {\n\t\t\t\tyears,\n\t\t\t\tmonths,\n\t\t\t\tweeks,\n\t\t\t\tdays,\n\t\t\t\thours,\n\t\t\t\tminutes,\n\t\t\t\tseconds,\n\t\t\t}\n\t\t}))\n}\n\n/// Parses the ISO 8601 Duration standard\n/// https://en.wikipedia.org/wiki/ISO_8601#Durations\nfn main() {\n\tlet input = \"P3Y6M4DT12H30M5S\";\n\tlet result = parser().parse(input.as_bytes());\n\n\tassert_eq!(\n\t\tDuration {\n\t\t\tyears: Some(3f32),\n\t\t\tmonths: Some(6f32),\n\t\t\tweeks: None,\n\t\t\tdays: Some(4f32),\n\t\t\thours: Some(12f32),\n\t\t\tminutes: Some(30f32),\n\t\t\tseconds: Some(5f32)\n\t\t},\n\t\tresult.unwrap()\n\t);\n}\n" }, { "path": "examples/json.rs", "content": "use pom::char_class::hex_digit;\nuse pom::parser::*;\n\nuse std::char::{decode_utf16, REPLACEMENT_CHARACTER};\nuse std::collections::HashMap;\nuse std::str::{self, FromStr};\n\n#[derive(Debug, PartialEq)]\npub enum JsonValue {\n\tNull,\n\tBool(bool),\n\tStr(String),\n\tNum(f64),\n\tArray(Vec),\n\tObject(HashMap),\n}\n\nfn space<'a>() -> Parser<'a, u8, ()> {\n\tone_of(b\" \\t\\r\\n\").repeat(0..).discard()\n}\n\nfn number<'a>() -> Parser<'a, u8, f64> {\n\tlet integer = one_of(b\"123456789\") - one_of(b\"0123456789\").repeat(0..) | sym(b'0');\n\tlet frac = sym(b'.') + one_of(b\"0123456789\").repeat(1..);\n\tlet exp = one_of(b\"eE\") + one_of(b\"+-\").opt() + one_of(b\"0123456789\").repeat(1..);\n\tlet number = sym(b'-').opt() + integer + frac.opt() + exp.opt();\n\tnumber\n\t\t.collect()\n\t\t.convert(str::from_utf8)\n\t\t.convert(f64::from_str)\n}\n\nfn string<'a>() -> Parser<'a, u8, String> {\n\tlet special_char = sym(b'\\\\')\n\t\t| sym(b'/')\n\t\t| sym(b'\"')\n\t\t| sym(b'b').map(|_| b'\\x08')\n\t\t| sym(b'f').map(|_| b'\\x0C')\n\t\t| sym(b'n').map(|_| b'\\n')\n\t\t| sym(b'r').map(|_| b'\\r')\n\t\t| sym(b't').map(|_| b'\\t');\n\tlet escape_sequence = sym(b'\\\\') * special_char;\n\tlet char_string = (none_of(b\"\\\\\\\"\") | escape_sequence)\n\t\t.repeat(1..)\n\t\t.convert(String::from_utf8);\n\tlet utf16_char = seq(b\"\\\\u\")\n\t\t* is_a(hex_digit)\n\t\t\t.repeat(4)\n\t\t\t.convert(String::from_utf8)\n\t\t\t.convert(|digits| u16::from_str_radix(&digits, 16));\n\tlet utf16_string = utf16_char.repeat(1..).map(|chars| {\n\t\tdecode_utf16(chars)\n\t\t\t.map(|r| r.unwrap_or(REPLACEMENT_CHARACTER))\n\t\t\t.collect::()\n\t});\n\tlet string = sym(b'\"') * (char_string | utf16_string).repeat(0..) - sym(b'\"');\n\tstring.map(|strings| strings.concat())\n}\n\nfn array<'a>() -> Parser<'a, u8, Vec> {\n\tlet elems = list(call(value), sym(b',') * space());\n\tsym(b'[') * space() * elems - sym(b']')\n}\n\nfn object<'a>() -> Parser<'a, u8, HashMap> {\n\tlet member = string() - space() - sym(b':') - space() + call(value);\n\tlet members = list(member, sym(b',') * space());\n\tlet obj = sym(b'{') * space() * members - sym(b'}');\n\tobj.map(|members| members.into_iter().collect::>())\n}\n\nfn value<'a>() -> Parser<'a, u8, JsonValue> {\n\t(seq(b\"null\").map(|_| JsonValue::Null)\n\t\t| seq(b\"true\").map(|_| JsonValue::Bool(true))\n\t\t| seq(b\"false\").map(|_| JsonValue::Bool(false))\n\t\t| number().map(|num| JsonValue::Num(num))\n\t\t| string().map(|text| JsonValue::Str(text))\n\t\t| array().map(|arr| JsonValue::Array(arr))\n\t\t| object().map(|obj| JsonValue::Object(obj)))\n\t\t- space()\n}\n\npub fn json<'a>() -> Parser<'a, u8, JsonValue> {\n\tspace() * value() - end()\n}\n\n#[allow(dead_code)]\nfn main() {\n\tlet input = br#\"\n\t{\n \"Image\": {\n \"Width\": 800,\n \"Height\": 600,\n \"Title\": \"View from 15th Floor\",\n \"Thumbnail\": {\n \"Url\": \"http://www.example.com/image/481989943\",\n \"Height\": 125,\n \"Width\": 100\n },\n \"Animated\" : false,\n \"IDs\": [116, 943, 234, 38793]\n },\n \"escaped characters\": \"\\u2192\\uD83D\\uDE00\\\"\\t\\uD834\\uDD1E\"\n }\"#;\n\n\tprintln!(\"{:?}\", json().parse(input));\n}\n" }, { "path": "examples/json_char.rs", "content": "use pom::parser::*;\n\nuse std::char::{decode_utf16, REPLACEMENT_CHARACTER};\nuse std::collections::HashMap;\nuse std::iter::FromIterator;\nuse std::str::FromStr;\n\n#[derive(Debug, PartialEq)]\npub enum JsonValue {\n\tNull,\n\tBool(bool),\n\tStr(String),\n\tNum(f64),\n\tArray(Vec),\n\tObject(HashMap),\n}\n\nfn space<'a>() -> Parser<'a, char, ()> {\n\tone_of(\" \\t\\r\\n\").repeat(0..).discard()\n}\n\nfn number<'a>() -> Parser<'a, char, f64> {\n\tlet integer = one_of(\"123456789\") - one_of(\"0123456789\").repeat(0..) | sym('0');\n\tlet frac = sym('.') + one_of(\"0123456789\").repeat(1..);\n\tlet exp = one_of(\"eE\") + one_of(\"+-\").opt() + one_of(\"0123456789\").repeat(1..);\n\tlet number = sym('-').opt() + integer + frac.opt() + exp.opt();\n\tnumber\n\t\t.collect()\n\t\t.map(String::from_iter)\n\t\t.convert(|s| f64::from_str(&s))\n}\n\nfn string<'a>() -> Parser<'a, char, String> {\n\tlet special_char = sym('\\\\')\n\t\t| sym('/')\n\t\t| sym('\"')\n\t\t| sym('b').map(|_| '\\x08')\n\t\t| sym('f').map(|_| '\\x0C')\n\t\t| sym('n').map(|_| '\\n')\n\t\t| sym('r').map(|_| '\\r')\n\t\t| sym('t').map(|_| '\\t');\n\tlet escape_sequence = sym('\\\\') * special_char;\n\tlet char_string = (none_of(\"\\\\\\\"\") | escape_sequence)\n\t\t.repeat(1..)\n\t\t.map(String::from_iter);\n\tlet utf16_char = tag(\"\\\\u\")\n\t\t* is_a(|c: char| c.is_digit(16))\n\t\t\t.repeat(4)\n\t\t\t.map(String::from_iter)\n\t\t\t.convert(|digits| u16::from_str_radix(&digits, 16));\n\tlet utf16_string = utf16_char.repeat(1..).map(|chars| {\n\t\tdecode_utf16(chars)\n\t\t\t.map(|r| r.unwrap_or(REPLACEMENT_CHARACTER))\n\t\t\t.collect::()\n\t});\n\tlet string = sym('\"') * (char_string | utf16_string).repeat(0..) - sym('\"');\n\tstring.map(|strings| strings.concat())\n}\n\nfn array<'a>() -> Parser<'a, char, Vec> {\n\tlet elems = list(call(value), sym(',') * space());\n\tsym('[') * space() * elems - sym(']')\n}\n\nfn object<'a>() -> Parser<'a, char, HashMap> {\n\tlet member = string() - space() - sym(':') - space() + call(value);\n\tlet members = list(member, sym(',') * space());\n\tlet obj = sym('{') * space() * members - sym('}');\n\tobj.map(|members| members.into_iter().collect::>())\n}\n\nfn value<'a>() -> Parser<'a, char, JsonValue> {\n\t(tag(\"null\").map(|_| JsonValue::Null)\n\t\t| tag(\"true\").map(|_| JsonValue::Bool(true))\n\t\t| tag(\"false\").map(|_| JsonValue::Bool(false))\n\t\t| number().map(|num| JsonValue::Num(num))\n\t\t| string().map(|text| JsonValue::Str(text))\n\t\t| array().map(|arr| JsonValue::Array(arr))\n\t\t| object().map(|obj| JsonValue::Object(obj)))\n\t\t- space()\n}\n\npub fn json<'a>() -> Parser<'a, char, JsonValue> {\n\tspace() * value() - end()\n}\n\n#[allow(dead_code)]\nfn main() {\n\tlet test = r#\"\n\t{\n \"Image\": {\n \"Width\": 800,\n \"Height\": 600,\n \"Title\": \"View from 15th Floor\",\n \"Thumbnail\": {\n \"Url\": \"http://www.example.com/image/481989943\",\n \"Height\": 125,\n \"Width\": 100\n },\n \"Animated\" : false,\n \"IDs\": [116, 943, 234, 38793]\n },\n \"escaped characters\": \"\\u2192\\uD83D\\uDE00\\\"\\t\\uD834\\uDD1E\"\n }\"#;\n\n\tlet input: Vec = test.chars().collect();\n\tprintln!(\"{:?}\", json().parse(&input));\n}\n" }, { "path": "examples/json_file.rs", "content": "use pom::char_class::hex_digit;\nuse pom::parser::{call, end, is_a, list, none_of, one_of, seq, sym, Parser};\n\nuse std::char::{decode_utf16, REPLACEMENT_CHARACTER};\nuse std::collections::HashMap;\nuse std::fs::File;\nuse std::io::Read;\nuse std::str::{self, FromStr};\n\n#[derive(Debug, PartialEq)]\npub enum JsonValue {\n\tNull,\n\tBool(bool),\n\tStr(String),\n\tNum(f64),\n\tArray(Vec),\n\tObject(HashMap),\n}\n\nfn space<'a>() -> Parser<'a, u8, ()> {\n\tone_of(b\" \\t\\r\\n\").repeat(0..).discard()\n}\n\nfn number<'a>() -> Parser<'a, u8, f64> {\n\tlet integer = one_of(b\"123456789\") - one_of(b\"0123456789\").repeat(0..) | sym(b'0');\n\tlet frac = sym(b'.') + one_of(b\"0123456789\").repeat(1..);\n\tlet exp = one_of(b\"eE\") + one_of(b\"+-\").opt() + one_of(b\"0123456789\").repeat(1..);\n\tlet number = sym(b'-').opt() + integer + frac.opt() + exp.opt();\n\tnumber\n\t\t.collect()\n\t\t.convert(str::from_utf8)\n\t\t.convert(f64::from_str)\n}\n\nfn string<'a>() -> Parser<'a, u8, String> {\n\tlet special_char = sym(b'\\\\')\n\t\t| sym(b'/')\n\t\t| sym(b'\"')\n\t\t| sym(b'b').map(|_| b'\\x08')\n\t\t| sym(b'f').map(|_| b'\\x0C')\n\t\t| sym(b'n').map(|_| b'\\n')\n\t\t| sym(b'r').map(|_| b'\\r')\n\t\t| sym(b't').map(|_| b'\\t');\n\tlet escape_sequence = sym(b'\\\\') * special_char;\n\tlet char_string = (none_of(b\"\\\\\\\"\") | escape_sequence)\n\t\t.repeat(1..)\n\t\t.convert(String::from_utf8);\n\tlet utf16_char = seq(b\"\\\\u\")\n\t\t* is_a(hex_digit)\n\t\t\t.repeat(4)\n\t\t\t.convert(String::from_utf8)\n\t\t\t.convert(|digits| u16::from_str_radix(&digits, 16));\n\tlet utf16_string = utf16_char.repeat(1..).map(|chars| {\n\t\tdecode_utf16(chars)\n\t\t\t.map(|r| r.unwrap_or(REPLACEMENT_CHARACTER))\n\t\t\t.collect::()\n\t});\n\tlet string = sym(b'\"') * (char_string | utf16_string).repeat(0..) - sym(b'\"');\n\tstring.map(|strings| strings.concat())\n}\n\nfn array<'a>() -> Parser<'a, u8, Vec> {\n\tlet elems = list(call(value), sym(b',') * space());\n\tsym(b'[') * space() * elems - sym(b']')\n}\n\nfn object<'a>() -> Parser<'a, u8, HashMap> {\n\tlet member = string() - space() - sym(b':') - space() + call(value);\n\tlet members = list(member, sym(b',') * space());\n\tlet obj = sym(b'{') * space() * members - sym(b'}');\n\tobj.map(|members| members.into_iter().collect::>())\n}\n\nfn value<'a>() -> Parser<'a, u8, JsonValue> {\n\t(seq(b\"null\").map(|_| JsonValue::Null)\n\t\t| seq(b\"true\").map(|_| JsonValue::Bool(true))\n\t\t| seq(b\"false\").map(|_| JsonValue::Bool(false))\n\t\t| number().map(|num| JsonValue::Num(num))\n\t\t| string().map(|text| JsonValue::Str(text))\n\t\t| array().map(|arr| JsonValue::Array(arr))\n\t\t| object().map(|obj| JsonValue::Object(obj)))\n\t\t- space()\n}\n\npub fn json<'a>() -> Parser<'a, u8, JsonValue> {\n\tspace() * value() - end()\n}\n\n#[allow(dead_code)]\nfn main() {\n\tlet mut file = File::open(\"examples/test.json\").unwrap();\n\tlet mut input: Vec = Vec::new();\n\tfile.read_to_end(&mut input).expect(\"read test.json\");\n\tprintln!(\"{:?}\", json().parse(input.as_slice()));\n}\n" }, { "path": "examples/simple.rs", "content": "use pom::parser::*;\n\nfn main() {\n\tlet input = b\"abcde\";\n\tlet parser = sym(b'a') * none_of(b\"AB\") - sym(b'c') + seq(b\"de\");\n\tlet output = parser.parse(input);\n\t// assert_eq!(output, Ok( (b'b', &b\"de\"[..]) ) );\n\tprintln!(\"{:?}\", output);\n}\n" }, { "path": "examples/test.json", "content": "{\n\t\"Image\": {\n\t\t\"Width\": 800,\n\t\t\"Height\": 600,\n\t\t\"Title\": \"View from 15th Floor\",\n\t\t\"Thumbnail\": {\n\t\t\t\"Url\": \"http://www.example.com/image/481989943\",\n\t\t\t\"Height\": 125,\n\t\t\t\"Width\": 100\n\t\t},\n\t\t\"Animated\" : false,\n\t\t\"IDs\": [116, 943, 234, 38793]\n\t},\n\t\"escaped characters\": \"\\u2192\\uD83D\\uDE00\\\"\\t\\uD834\\uDD1E\"\n}\n" }, { "path": "examples/utf8.rs", "content": "// Example shows basic UTF-8 combinators\n\nuse pom::utf8::*;\n\nfn main() {\n\t// Informal, Spanish-language movie database format\n\tlet input = \"\\\nTítulo: Abre los ojos\nAño: 1997\nDirector: Alejandro Amenábar\n\nTítulo: Amores Perros\nDirector: Alejandro González Iñárritu\nAño: 2000\n\nTítulo: La montaña sagrada\nAño: 1973\nDirector: Alejandro Jodorowsky\n\";\n\n\tenum DataLine<'a> {\n\t\tTitle(&'a str),\n\t\tDirector(&'a str),\n\t\tYear(i32),\n\t}\n\n\tfn positive<'a>() -> Parser<'a, i32> {\n\t\t//\t\tlet integer = (one_of(\"123456789\") - one_of(\"0123456789\").repeat(0..)) | sym(b'0'); // TODO\n\t\tlet digit = one_of(\"0123456789\");\n\t\tlet integer = digit.discard().repeat(1..);\n\t\tinteger.collect().convert(|x| x.parse::())\n\t}\n\n\tfn rest_str<'a>() -> Parser<'a, &'a str> {\n\t\tany().repeat(1..).collect()\n\t}\n\n\tfn separator<'a>() -> Parser<'a, ()> {\n\t\tseq(\": \").discard()\n\t}\n\n\tlet parser = (seq(\"Título\") * separator() * rest_str().map(|s| DataLine::Title(s)))\n\t\t| (seq(\"Director\") * separator() * rest_str().map(|s| DataLine::Director(s)))\n\t\t| (seq(\"Año\") * separator() * positive().map(|i| DataLine::Year(i)));\n\n\t{\n\t\tlet mut title_opt: Option<&str> = None;\n\t\tlet mut year_opt: Option = None;\n\t\tlet mut director_opt: Option<&str> = None;\n\n\t\tfor line in input.lines() {\n\t\t\tif !line.is_empty() {\n\t\t\t\t// Skip blank lines without parsing\n\t\t\t\t// Parse line\n\t\t\t\tmatch parser.parse_str(line).unwrap() {\n\t\t\t\t\tDataLine::Title(s) => title_opt = Some(s),\n\t\t\t\t\tDataLine::Director(s) => director_opt = Some(s),\n\t\t\t\t\tDataLine::Year(s) => year_opt = Some(s),\n\t\t\t\t}\n\t\t\t\t// When all three line types have been collected, print them\n\t\t\t\tif let (Some(title), Some(year), Some(director)) =\n\t\t\t\t\t(title_opt, year_opt, director_opt)\n\t\t\t\t{\n\t\t\t\t\tprintln!(\"Title: {}\\nDirector: {}\\nYear: {}\\n\", title, director, year);\n\t\t\t\t\t(title_opt, year_opt, director_opt) = (None, None, None);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "examples/utf8_mixed.rs", "content": "// Example shows UTF-8 combinators intermixed with binary combinators\n\nuse pom::parser::*;\nuse pom::utf8;\n\nfn main() {\n\t// A parser for MsgPack (but only messages encoding a string)\n\tlet testcases: [Vec; 6] = [\n\t\tvec![0b10100100, 0b11110000, 0b10011111, 0b10100100, 0b10010100], // 🤔, max-size 31 format\n\t\tvec![0xd9, 4, 0b11110000, 0b10011111, 0b10011000, 0b10101110], // 😮, max-size 255 format\n\t\tvec![0xda, 0, 4, 0b11110000, 0b10011111, 0b10100100, 0b10101111], // 🤯, max-size 2^16-1 format\n\t\tvec![\n\t\t\t0xdb, 0, 0, 0, 4, 0b11110000, 0b10011111, 0b10010010, 0b10100101,\n\t\t], // 💥, max-size 2^32-1 format\n\t\tvec![0xc4, 4, 0b11110000, 0b10011111, 0b10011000, 0b10101110], // Valid MsgPack, but not a string (binary)\n\t\tvec![0b10100100, 0b10010100, 0b10100100, 0b10011111, 0b11110000], // A MsgPack string, but invalid UTF-8\n\t];\n\n\tconst MASK: u8 = 0b11100000; // size 31 format is denoted by 3 high bits == 101\n\tconst SIZE_31: u8 = 0b10100000;\n\n\tfn rest_as_str<'a>() -> utf8::Parser<'a, &'a str> {\n\t\tutf8::any().repeat(0..).collect()\n\t}\n\n\t// Demo parser does not verify that the claimed length matches the actual length (but checking so is simple with >>)\n\tlet parser = (sym(0xdb) * any().repeat(4) * rest_as_str()) // 2^32-1 format\n\t\t| (sym(0xda) * any().repeat(2) * rest_as_str()) // 2^16-1 format\n\t\t| (sym(0xd9) * any() * rest_as_str()) // 255 format\n\t\t| (is_a(|x| x&MASK == SIZE_31) * rest_as_str()) // 31 format\n\t\t- end();\n\n\tfor testcase in testcases.iter() {\n\t\tprintln!(\"{:?}\", parser.parse(testcase));\n\t}\n}\n" }, { "path": "examples/whitespace.rs", "content": "use pom::parser::*;\n\n#[derive(Clone, Debug, PartialEq)]\nstruct Container {\n\tcontainers: Vec,\n\tcontents: Vec,\n}\n\nenum TmpContainerOrContent {\n\tContainer(Container),\n\tContent(String),\n}\n\nfn whitespace<'a>() -> Parser<'a, u8, ()> {\n\tone_of(b\" \\t\\r\\n\").repeat(0..).discard()\n}\n\nfn linebreak<'a>() -> Parser<'a, u8, ()> {\n\tsym(b'\\r').opt() * sym(b'\\n').discard()\n}\n\nfn indented<'a>() -> Parser<'a, u8, Vec> {\n\tsym(b'\\t') * none_of(b\"\\n\\r\").repeat(1..) - linebreak()\n}\n\nfn empty<'a>() -> Parser<'a, u8, ()> {\n\tone_of(b\" \\t\").repeat(0..).discard() - linebreak()\n}\n\nfn content<'a>() -> Parser<'a, u8, String> {\n\tnone_of(b\" \\t\\r\\n\").repeat(1..).convert(String::from_utf8) - linebreak()\n}\n\nfn subcontainer<'a>() -> Parser<'a, u8, (Vec, Vec)> {\n\t(call(container).map(|ctr| TmpContainerOrContent::Container(ctr))\n\t\t| content().map(|ctn| TmpContainerOrContent::Content(ctn)))\n\t.repeat(1..)\n\t.map(|tmp| {\n\t\ttmp.into_iter().fold((vec![], vec![]), |acc, x| match x {\n\t\t\tTmpContainerOrContent::Container(ct) => (\n\t\t\t\tacc.0.into_iter().chain(vec![ct].into_iter()).collect(),\n\t\t\t\tacc.1,\n\t\t\t),\n\t\t\tTmpContainerOrContent::Content(cn) => (\n\t\t\t\tacc.0,\n\t\t\t\tacc.1.into_iter().chain(vec![cn].into_iter()).collect(),\n\t\t\t),\n\t\t})\n\t})\n}\n\nfn container<'a>() -> Parser<'a, u8, Container> {\n\tseq(b\"Container\\n\")\n\t\t* (indented() | empty().map(|()| vec![]))\n\t\t\t.repeat(1..)\n\t\t\t.map(|lines| {\n\t\t\t\tlines\n\t\t\t\t\t.into_iter()\n\t\t\t\t\t.filter(|line| line.len() > 0)\n\t\t\t\t\t.fold(vec![], |accum, line| {\n\t\t\t\t\t\taccum\n\t\t\t\t\t\t\t.into_iter()\n\t\t\t\t\t\t\t.chain(line.into_iter().chain(vec![b'\\n'].into_iter()))\n\t\t\t\t\t\t\t.collect()\n\t\t\t\t\t})\n\t\t\t})\n\t\t\t.map(|deden| subcontainer().parse(&deden).expect(\"subcont\"))\n\t\t\t.map(|(containers, contents)| Container {\n\t\t\t\tcontainers,\n\t\t\t\tcontents,\n\t\t\t})\n}\n\nfn mylang<'a>() -> Parser<'a, u8, Vec> {\n\twhitespace() * list(call(container), whitespace())\n}\n\nfn main() -> Result<(), ()> {\n\tlet input = br#\"\nContainer\n\tContainer\n\t\ta\n\t\tb\n\t\tc\n\n\t1\n\t2\n\t3\n\n\tContainer\n\t\tq\n\nContainer\n\tfoo\n\tbar\n\n\tContainer\n\t\tbaz\n\t\tquux\n\t\t\"#;\n\n\tassert_eq!(\n\t\tmylang().parse(input),\n\t\tOk(vec![\n\t\t\tContainer {\n\t\t\t\tcontainers: vec![\n\t\t\t\t\tContainer {\n\t\t\t\t\t\tcontainers: vec![],\n\t\t\t\t\t\tcontents: vec![\"a\".into(), \"b\".into(), \"c\".into(),]\n\t\t\t\t\t},\n\t\t\t\t\tContainer {\n\t\t\t\t\t\tcontainers: vec![],\n\t\t\t\t\t\tcontents: vec![\"q\".into(),]\n\t\t\t\t\t}\n\t\t\t\t],\n\t\t\t\tcontents: vec![\"1\".into(), \"2\".into(), \"3\".into(),]\n\t\t\t},\n\t\t\tContainer {\n\t\t\t\tcontainers: vec![Container {\n\t\t\t\t\tcontents: vec![\"baz\".into(), \"quux\".into(),],\n\t\t\t\t\tcontainers: vec![],\n\t\t\t\t},],\n\t\t\t\tcontents: vec![\"foo\".into(), \"bar\".into(),]\n\t\t\t},\n\t\t])\n\t);\n\n\tOk(())\n}\n" }, { "path": "rustfmt.toml", "content": "format_strings = false\nreorder_imports = true\nhard_tabs = true\n" }, { "path": "src/char_class.rs", "content": "/// Recognises an alphabetic character, `a-zA-Z`.\n#[inline]\npub fn alpha(term: u8) -> bool {\n\tterm.is_ascii_alphabetic()\n}\n\n/// Recognises an alphabetic character, `A-Z`.\n#[inline]\npub fn alpha_uppercase(term: u8) -> bool {\n\tterm.is_ascii_uppercase()\n}\n\n/// Recognises an alphabetic character, `a-z`.\n#[inline]\npub fn alpha_lowercase(term: u8) -> bool {\n\tterm.is_ascii_lowercase()\n}\n\n/// Recognises a decimal digit, `0-9`.\n#[inline]\npub fn digit(term: u8) -> bool {\n\tterm.is_ascii_digit()\n}\n\n/// Recognises an alphanumeric character, `a-zA-Z0-9`.\n#[inline]\npub fn alphanum(term: u8) -> bool {\n\tterm.is_ascii_alphanumeric()\n}\n\n/// Recognises a hexadecimal digit, `0-9a-fA-F`.\n#[inline]\npub fn hex_digit(term: u8) -> bool {\n\tmatches!(term, 0x30..=0x39 | 0x41..=0x46 | 0x61..=0x66)\n}\n\n/// Recognises an octal digit, `0-7`.\n#[inline]\npub fn oct_digit(term: u8) -> bool {\n\tmatches!(term, 0x30..=0x37)\n}\n\n/// Recognises a space or tab.\n#[inline]\npub fn space(term: u8) -> bool {\n\tmatches!(term, b' ' | b'\\t')\n}\n\n/// Recognises a space, tab, line feed, or carriage return.\n#[inline]\npub fn multispace(term: u8) -> bool {\n\tspace(term) || matches!(term, b'\\n' | b'\\r')\n}\n\n#[cfg(test)]\nmod test {\n\tuse super::*;\n\n\t#[test]\n\tfn is_an_alpha() {\n\t\tassert!(alpha(b'A'));\n\t\tassert!(alpha(b'Z'));\n\t\tassert!(alpha(b'a'));\n\t\tassert!(alpha(b'z'));\n\t}\n\n\t#[test]\n\tfn is_an_alpha_uppercase() {\n\t\tassert!(alpha_uppercase(b'A'));\n\t\tassert!(alpha_uppercase(b'Z'));\n\t\tassert!(!alpha_uppercase(b'a'));\n\t\tassert!(!alpha_uppercase(b'z'));\n\t}\n\n\t#[test]\n\tfn is_an_alpha_lowercase() {\n\t\tassert!(!alpha_lowercase(b'A'));\n\t\tassert!(!alpha_lowercase(b'Z'));\n\t\tassert!(alpha_lowercase(b'a'));\n\t\tassert!(alpha_lowercase(b'z'));\n\t}\n\n\t#[test]\n\tfn is_a_digit() {\n\t\tassert!(digit(b'0'));\n\t\tassert!(digit(b'9'));\n\t\tassert!(!digit(b'A'));\n\t}\n\n\t#[test]\n\tfn is_an_alphanum() {\n\t\tassert!(alphanum(b'A'));\n\t\tassert!(alphanum(b'Z'));\n\t\tassert!(alphanum(b'a'));\n\t\tassert!(alphanum(b'z'));\n\t\tassert!(alphanum(b'0'));\n\t\tassert!(alphanum(b'9'));\n\t\tassert!(!alphanum(b'#'));\n\t}\n\n\t#[test]\n\tfn is_a_hex_digit() {\n\t\tassert!(hex_digit(b'0'));\n\t\tassert!(hex_digit(b'9'));\n\t\tassert!(hex_digit(b'A'));\n\t\tassert!(hex_digit(b'F'));\n\t\tassert!(hex_digit(b'a'));\n\t\tassert!(hex_digit(b'f'));\n\t\tassert!(!hex_digit(b'G'));\n\t}\n\n\t#[test]\n\tfn is_a_oct_digit() {\n\t\tassert!(oct_digit(b'0'));\n\t\tassert!(oct_digit(b'7'));\n\t\tassert!(!oct_digit(b'8'));\n\t\tassert!(!oct_digit(b'9'));\n\t}\n\n\t#[test]\n\tfn is_space() {\n\t\tassert!(space(b' '));\n\t\tassert!(space(b'\\t'));\n\t\tassert!(!space(b'\\n'));\n\t\tassert!(!space(b'A'));\n\t}\n\n\t#[test]\n\tfn is_multispace() {\n\t\tassert!(multispace(b' '));\n\t\tassert!(multispace(b'\\t'));\n\t\tassert!(multispace(b'\\n'));\n\t\tassert!(!multispace(b'A'));\n\t}\n}" }, { "path": "src/lib.rs", "content": "pub(crate) mod range;\nmod result;\npub(crate) mod set;\n\n/// Contains predefined parsers and combinators.\npub mod parser;\n\n/// Utility functions to recognize char class of byte value.\npub mod char_class;\n\n/// Variants of parser functions specialized for matching UTF-8 strings and returning chars.\n/// Method and constructor names/functionality are generally the same as in base parser module.\n#[cfg(feature = \"utf8\")]\npub mod utf8;\n\npub use crate::result::{Error, Result};\n\n/// Parser type, `Parser` is alias of `parser::Parser<'static, I, O>`.\npub type Parser = parser::Parser<'static, I, O>;\n" }, { "path": "src/parser.rs", "content": "use super::{Error, Result};\nuse crate::{range::RangeArgument, set::Set};\nuse std::{\n\tfmt::{Debug, Display},\n\tops::Bound::{Excluded, Included, Unbounded},\n\tops::{Add, BitOr, Mul, Neg, Not, Shr, Sub},\n};\n\ntype Parse<'a, I, O> = dyn Fn(&'a [I], usize) -> Result<(O, usize)> + 'a;\n\n/// Parser combinator.\npub struct Parser<'a, I, O> {\n\tpub method: Box>,\n}\n\nimpl<'a, I, O> Parser<'a, I, O> {\n\t/// Create new parser.\n\tpub fn new

(parse: P) -> Self\n\twhere\n\t\tP: Fn(&'a [I], usize) -> Result<(O, usize)> + 'a,\n\t{\n\t\tSelf {\n\t\t\tmethod: Box::new(parse),\n\t\t}\n\t}\n\n\t/// Apply the parser to parse input.\n\tpub fn parse(&self, input: &'a [I]) -> Result {\n\t\t(self.method)(input, 0).map(|(out, _)| out)\n\t}\n\n\t/// Parse input at specified position.\n\tpub fn parse_at(&self, input: &'a [I], start: usize) -> Result<(O, usize)> {\n\t\t(self.method)(input, start)\n\t}\n\n\t/// Convert parser result to desired value.\n\tpub fn map(self, f: F) -> Parser<'a, I, U>\n\twhere\n\t\tF: Fn(O) -> U + 'a,\n\t\tI: 'a,\n\t\tO: 'a,\n\t\tU: 'a,\n\t{\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start).map(|(out, pos)| (f(out), pos))\n\t\t})\n\t}\n\n\t/// Convert parser result to desired value, fail in case of conversion error.\n\tpub fn convert(self, f: F) -> Parser<'a, I, U>\n\twhere\n\t\tF: Fn(O) -> ::std::result::Result + 'a,\n\t\tE: Debug,\n\t\tO: 'a,\n\t\tU: 'a,\n\t{\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start).and_then(|(res, pos)| match f(res) {\n\t\t\t\tOk(out) => Ok((out, pos)),\n\t\t\t\tErr(err) => Err(Error::Conversion {\n\t\t\t\t\tmessage: format!(\"Conversion error: {:?}\", err),\n\t\t\t\t\tposition: start,\n\t\t\t\t}),\n\t\t\t})\n\t\t})\n\t}\n\n\t/// Cache parser output result to speed up backtracking.\n\tpub fn cache(self) -> Self\n\twhere\n\t\tO: Clone + 'a,\n\t{\n\t\tuse std::{cell::RefCell, collections::HashMap};\n\t\tlet results = RefCell::new(HashMap::new());\n\t\tSelf::new(move |input: &'a [I], start: usize| {\n\t\t\tlet key = (start, format!(\"{:p}\", &self.method));\n\t\t\tresults\n\t\t\t\t.borrow_mut()\n\t\t\t\t.entry(key)\n\t\t\t\t.or_insert_with(|| (self.method)(input, start))\n\t\t\t\t.clone()\n\t\t})\n\t}\n\n\t/// Get input position after matching parser.\n\tpub fn pos(self) -> Parser<'a, I, usize>\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start).map(|(_, pos)| (pos, pos))\n\t\t})\n\t}\n\n\t/// Collect all matched input symbols.\n\tpub fn collect(self) -> Parser<'a, I, &'a [I]>\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start).map(|(_, end)| (&input[start..end], end))\n\t\t})\n\t}\n\n\t/// Discard parser output.\n\tpub fn discard(self) -> Parser<'a, I, ()>\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start).map(|(_, end)| ((), end))\n\t\t})\n\t}\n\n\t/// Make parser optional.\n\tpub fn opt(self) -> Parser<'a, I, Option>\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser::new(\n\t\t\tmove |input: &'a [I], start: usize| match (self.method)(input, start) {\n\t\t\t\tOk((out, pos)) => Ok((Some(out), pos)),\n\t\t\t\tErr(_) => Ok((None, start)),\n\t\t\t},\n\t\t)\n\t}\n\n\t/// `p.repeat(5)` repeat p exactly 5 times\n\t/// `p.repeat(0..)` repeat p zero or more times\n\t/// `p.repeat(1..)` repeat p one or more times\n\t/// `p.repeat(1..4)` match p at least 1 and at most 3 times\n\tpub fn repeat(self, range: R) -> Parser<'a, I, Vec>\n\twhere\n\t\tR: RangeArgument + Debug + 'a,\n\t\tO: 'a,\n\t{\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\tlet mut items = vec![];\n\t\t\tlet mut pos = start;\n\t\t\tloop {\n\t\t\t\tmatch range.end() {\n\t\t\t\t\tIncluded(&max_count) => {\n\t\t\t\t\t\tif items.len() >= max_count {\n\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tExcluded(&max_count) => {\n\t\t\t\t\t\tif items.len() + 1 >= max_count {\n\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tUnbounded => (),\n\t\t\t\t}\n\n\t\t\t\tlet Ok((item, item_pos)) = (self.method)(input, pos) else {\n\t\t\t\t\tbreak;\n\t\t\t\t};\n\t\t\t\titems.push(item);\n\t\t\t\tpos = item_pos;\n\t\t\t}\n\t\t\tif let Included(&min_count) = range.start() {\n\t\t\t\tif items.len() < min_count {\n\t\t\t\t\treturn Err(Error::Mismatch {\n\t\t\t\t\t\tmessage: format!(\n\t\t\t\t\t\t\t\"expect repeat at least {} times, found {} times\",\n\t\t\t\t\t\t\tmin_count,\n\t\t\t\t\t\t\titems.len()\n\t\t\t\t\t\t),\n\t\t\t\t\t\tposition: start,\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t}\n\t\t\tOk((items, pos))\n\t\t})\n\t}\n\n\t#[cfg(not(feature = \"trace\"))]\n\t/// Give parser a name to identify parsing errors.\n\tpub fn name(self, name: &'a str) -> Self\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser::new(\n\t\t\tmove |input: &'a [I], start: usize| match (self.method)(input, start) {\n\t\t\t\tres @ Ok(_) => res,\n\t\t\t\tErr(err) => match err {\n\t\t\t\t\tError::Custom { .. } => Err(err),\n\t\t\t\t\t_ => Err(Error::Custom {\n\t\t\t\t\t\tmessage: format!(\"failed to parse {}\", name),\n\t\t\t\t\t\tposition: start,\n\t\t\t\t\t\tinner: Some(Box::new(err)),\n\t\t\t\t\t}),\n\t\t\t\t},\n\t\t\t},\n\t\t)\n\t}\n\n\t#[cfg(feature = \"trace\")]\n\t/// Trace parser calls and results. Similar to name\n\tpub fn name(self, name: &'a str) -> Self\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\teprintln!(\"parse: {} ({})\", name, start);\n\t\t\tmatch (self.method)(input, start) {\n\t\t\t\tres @ Ok(_) => {\n\t\t\t\t\teprintln!(\" {} ({}): ok\", name, start);\n\t\t\t\t\tres\n\t\t\t\t}\n\t\t\t\tErr(err) => {\n\t\t\t\t\teprintln!(\" {} ({}): error\", name, start);\n\t\t\t\t\tmatch err {\n\t\t\t\t\t\tError::Custom { .. } => Err(err),\n\t\t\t\t\t\t_ => Err(Error::Custom {\n\t\t\t\t\t\t\tmessage: format!(\"failed to parse {}\", name),\n\t\t\t\t\t\t\tposition: start,\n\t\t\t\t\t\t\tinner: Some(Box::new(err)),\n\t\t\t\t\t\t}),\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\t/// Mark parser as expected, abort early when failed in ordered choice.\n\tpub fn expect(self, name: &'a str) -> Self\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser::new(\n\t\t\tmove |input: &'a [I], start: usize| match (self.method)(input, start) {\n\t\t\t\tres @ Ok(_) => res,\n\t\t\t\tErr(err) => Err(Error::Expect {\n\t\t\t\t\tmessage: format!(\"Expect {}\", name),\n\t\t\t\t\tposition: start,\n\t\t\t\t\tinner: Box::new(err),\n\t\t\t\t}),\n\t\t\t},\n\t\t)\n\t}\n}\n\n/// Always succeeds, consume no input.\npub fn empty<'a, I>() -> Parser<'a, I, ()> {\n\tParser::new(|_: &[I], start: usize| Ok(((), start)))\n}\n\n/// Match any symbol.\npub fn any<'a, I>() -> Parser<'a, I, I>\nwhere\n\tI: Clone,\n{\n\tParser::new(|input: &[I], start: usize| {\n\t\tlet Some(s) = input.get(start) else {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: \"end of input reached\".to_owned(),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t};\n\t\tOk((s.clone(), start + 1))\n\t})\n}\n\n/// Success when current input symbol equals `t`.\npub fn sym<'a, I>(t: I) -> Parser<'a, I, I>\nwhere\n\tI: Clone + PartialEq + Display,\n{\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet Some(s) = input.get(start) else {\n\t\t\treturn Err(Error::Incomplete);\n\t\t};\n\t\tif t != *s {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"expect: {}, found: {}\", t, s),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tOk((s.clone(), start + 1))\n\t})\n}\n\n/// Success when sequence of symbols matches current input.\npub fn seq<'a, 'b: 'a, I>(tag: &'b [I]) -> Parser<'a, I, &'a [I]>\nwhere\n\tI: PartialEq + Debug,\n{\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet mut index = 0;\n\t\tloop {\n\t\t\tlet pos = start + index;\n\t\t\tif index == tag.len() {\n\t\t\t\treturn Ok((tag, pos));\n\t\t\t}\n\t\t\tlet Some(s) = input.get(pos) else {\n\t\t\t\treturn Err(Error::Incomplete);\n\t\t\t};\n\t\t\tif tag[index] != *s {\n\t\t\t\treturn Err(Error::Mismatch {\n\t\t\t\t\tmessage: format!(\"seq {:?} expect: {:?}, found: {:?}\", tag, tag[index], s),\n\t\t\t\t\tposition: pos,\n\t\t\t\t});\n\t\t\t}\n\t\t\tindex += 1;\n\t\t}\n\t})\n}\n\n/// Success when tag matches current input.\npub fn tag<'a, 'b: 'a>(tag: &'b str) -> Parser<'a, char, &'a str> {\n\tParser::new(move |input: &'a [char], start: usize| {\n\t\tlet mut pos = start;\n\t\tfor c in tag.chars() {\n\t\t\tlet Some(&s) = input.get(pos) else {\n\t\t\t\treturn Err(Error::Incomplete);\n\t\t\t};\n\t\t\tif c != s {\n\t\t\t\treturn Err(Error::Mismatch {\n\t\t\t\t\tmessage: format!(\"tag {:?} expect: {:?}, found: {}\", tag, c, s),\n\t\t\t\t\tposition: pos,\n\t\t\t\t});\n\t\t\t}\n\t\t\tpos += 1;\n\t\t}\n\t\tOk((tag, pos))\n\t})\n}\n\n/// Parse separated list.\npub fn list<'a, I, O, U>(\n\tparser: Parser<'a, I, O>,\n\tseparator: Parser<'a, I, U>,\n) -> Parser<'a, I, Vec>\nwhere\n\tO: 'a,\n\tU: 'a,\n{\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet mut items = vec![];\n\t\tlet mut pos = start;\n\t\tif let Ok((first_item, first_pos)) = (parser.method)(input, pos) {\n\t\t\titems.push(first_item);\n\t\t\tpos = first_pos;\n\t\t\twhile let Ok((_, sep_pos)) = (separator.method)(input, pos) {\n\t\t\t\tmatch (parser.method)(input, sep_pos) {\n\t\t\t\t\tOk((more_item, more_pos)) => {\n\t\t\t\t\t\titems.push(more_item);\n\t\t\t\t\t\tpos = more_pos;\n\t\t\t\t\t}\n\t\t\t\t\tErr(_) => break,\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tOk((items, pos))\n\t})\n}\n\n/// Success when current input symbol is one of the set.\npub fn one_of<'a, I, S>(set: &'a S) -> Parser<'a, I, I>\nwhere\n\tI: Clone + PartialEq + Display + Debug,\n\tS: Set + ?Sized,\n{\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet Some(s) = input.get(start) else {\n\t\t\treturn Err(Error::Incomplete);\n\t\t};\n\t\tif !set.contains(s) {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"expect one of: {}, found: {}\", set.to_str(), s),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t};\n\t\tOk((s.clone(), start + 1))\n\t})\n}\n\n/// Success when current input symbol is none of the set.\npub fn none_of<'a, I, S>(set: &'static S) -> Parser<'a, I, I>\nwhere\n\tI: Clone + PartialEq + Display + Debug,\n\tS: Set + ?Sized,\n{\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet Some(s) = input.get(start) else {\n\t\t\treturn Err(Error::Incomplete);\n\t\t};\n\t\tif set.contains(s) {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"expect none of: {}, found: {}\", set.to_str(), s),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tOk((s.clone(), start + 1))\n\t})\n}\n\n/// Success when predicate returns true on current input symbol.\npub fn is_a<'a, I, F>(predicate: F) -> Parser<'a, I, I>\nwhere\n\tI: Clone + PartialEq + Display + Debug,\n\tF: Fn(I) -> bool + 'a,\n{\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet Some(s) = input.get(start) else {\n\t\t\treturn Err(Error::Incomplete);\n\t\t};\n\t\tif !predicate(s.clone()) {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"is_a predicate failed on: {}\", s),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tOk((s.clone(), start + 1))\n\t})\n}\n\n/// Success when predicate returns false on current input symbol.\npub fn not_a<'a, I, F>(predicate: F) -> Parser<'a, I, I>\nwhere\n\tI: Clone + PartialEq + Display + Debug,\n\tF: Fn(I) -> bool + 'a,\n{\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet Some(s) = input.get(start) else {\n\t\t\treturn Err(Error::Incomplete);\n\t\t};\n\t\tif predicate(s.clone()) {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"not_a predicate failed on: {}\", s),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tOk((s.clone(), start + 1))\n\t})\n}\n\n/// Read n symbols.\npub fn take<'a, I>(n: usize) -> Parser<'a, I, &'a [I]> {\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet pos = start + n;\n\t\tif input.len() < pos {\n\t\t\treturn Err(Error::Incomplete);\n\t\t}\n\t\tOk((&input[start..pos], pos))\n\t})\n}\n\n/// Skip n symbols.\npub fn skip<'a, I>(n: usize) -> Parser<'a, I, ()> {\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet pos = start + n;\n\t\tif input.len() < pos {\n\t\t\treturn Err(Error::Incomplete);\n\t\t}\n\t\tOk(((), pos))\n\t})\n}\n\n/// Call a parser factory, can be used to create recursive parsers.\npub fn call<'a, I, O, F>(parser_factory: F) -> Parser<'a, I, O>\nwhere\n\tO: 'a,\n\tF: Fn() -> Parser<'a, I, O> + 'a,\n{\n\tParser::new(move |input: &'a [I], start: usize| {\n\t\tlet parser = parser_factory();\n\t\t(parser.method)(input, start)\n\t})\n}\n\n/// Success when end of input is reached.\npub fn end<'a, I>() -> Parser<'a, I, ()>\nwhere\n\tI: Display,\n{\n\tParser::new(|input: &'a [I], start: usize| {\n\t\tif let Some(s) = input.get(start) {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"expect end of input, found: {}\", s),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tOk(((), start))\n\t})\n}\n\n/// Sequence reserve value\nimpl<'a, I, O: 'a, U: 'a> Add> for Parser<'a, I, O> {\n\ttype Output = Parser<'a, I, (O, U)>;\n\n\tfn add(self, other: Parser<'a, I, U>) -> Self::Output {\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start).and_then(|(out1, pos1)| {\n\t\t\t\t(other.method)(input, pos1).map(|(out2, pos2)| ((out1, out2), pos2))\n\t\t\t})\n\t\t})\n\t}\n}\n\n/// Sequence discard second value\nimpl<'a, I, O: 'a, U: 'a> Sub> for Parser<'a, I, O> {\n\ttype Output = Parser<'a, I, O>;\n\n\tfn sub(self, other: Parser<'a, I, U>) -> Self::Output {\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start)\n\t\t\t\t.and_then(|(out1, pos1)| (other.method)(input, pos1).map(|(_, pos2)| (out1, pos2)))\n\t\t})\n\t}\n}\n\n/// Sequence discard first value\nimpl<'a, I: 'a, O: 'a, U: 'a> Mul> for Parser<'a, I, O> {\n\ttype Output = Parser<'a, I, U>;\n\n\tfn mul(self, other: Parser<'a, I, U>) -> Self::Output {\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start).and_then(|(_, pos1)| (other.method)(input, pos1))\n\t\t})\n\t}\n}\n\n/// Chain two parsers where the second parser depends on the first's result.\nimpl<'a, I, O: 'a, U: 'a, F: Fn(O) -> Parser<'a, I, U> + 'a> Shr for Parser<'a, I, O> {\n\ttype Output = Parser<'a, I, U>;\n\n\tfn shr(self, other: F) -> Self::Output {\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start).and_then(|(out, pos)| (other(out).method)(input, pos))\n\t\t})\n\t}\n}\n\n/// Ordered choice\nimpl<'a, I, O: 'a> BitOr for Parser<'a, I, O> {\n\ttype Output = Parser<'a, I, O>;\n\n\tfn bitor(self, other: Parser<'a, I, O>) -> Self::Output {\n\t\tParser::new(\n\t\t\tmove |input: &'a [I], start: usize| match (self.method)(input, start) {\n\t\t\t\tOk(out) => Ok(out),\n\t\t\t\tErr(err) => match err {\n\t\t\t\t\tError::Expect { .. } => Err(err),\n\t\t\t\t\t_ => (other.method)(input, start),\n\t\t\t\t},\n\t\t\t},\n\t\t)\n\t}\n}\n\n/// And predicate\nimpl<'a, I, O: 'a> Neg for Parser<'a, I, O> {\n\ttype Output = Parser<'a, I, bool>;\n\n\tfn neg(self) -> Self::Output {\n\t\tParser::new(move |input: &'a [I], start: usize| {\n\t\t\t(self.method)(input, start).map(|_| (true, start))\n\t\t})\n\t}\n}\n\n/// Not predicate\nimpl<'a, I, O: 'a> Not for Parser<'a, I, O> {\n\ttype Output = Parser<'a, I, bool>;\n\n\tfn not(self) -> Self::Output {\n\t\tParser::new(\n\t\t\tmove |input: &'a [I], start: usize| match (self.method)(input, start) {\n\t\t\t\tOk(_) => Err(Error::Mismatch {\n\t\t\t\t\tmessage: \"not predicate failed\".to_string(),\n\t\t\t\t\tposition: start,\n\t\t\t\t}),\n\t\t\t\tErr(_) => Ok((true, start)),\n\t\t\t},\n\t\t)\n\t}\n}\n\n#[cfg(test)]\nmod tests {\n\tuse crate::parser::*;\n\tuse crate::Error;\n\n\t#[test]\n\tfn byte_works() {\n\t\tlet input = b\"abcde\";\n\t\tlet parser = sym(b'a') + one_of(b\"ab\") - sym(b'C');\n\t\tlet output = parser.parse(input);\n\t\tassert_eq!(\n\t\t\toutput,\n\t\t\tErr(Error::Mismatch {\n\t\t\t\tmessage: \"expect: 67, found: 99\".to_string(),\n\t\t\t\tposition: 2\n\t\t\t})\n\t\t);\n\n\t\tlet parser = sym(b'a') * none_of(b\"AB\") - sym(b'c') + seq(b\"de\");\n\t\tlet output = parser.parse(input);\n\t\tassert_eq!(output, Ok((b'b', &b\"de\"[..])));\n\t\tassert_eq!(parser.pos().parse(input), Ok(5));\n\n\t\tlet parser = sym(b'e') | sym(b'd').expect(\"d\") | empty().map(|_| b'0');\n\t\tlet output = parser.parse(input);\n\t\tassert_eq!(\n\t\t\toutput,\n\t\t\tErr(Error::Expect {\n\t\t\t\tmessage: \"Expect d\".to_owned(),\n\t\t\t\tposition: 0,\n\t\t\t\tinner: Box::new(Error::Mismatch {\n\t\t\t\t\tmessage: \"expect: 100, found: 97\".to_string(),\n\t\t\t\t\tposition: 0\n\t\t\t\t})\n\t\t\t})\n\t\t);\n\t}\n\n\t#[test]\n\tfn char_works() {\n\t\tlet input = \"abcd\".chars().collect::>();\n\t\tlet parser = tag(\"ab\") + sym('c') | sym('d').map(|_| (\"\", '0'));\n\t\tlet output = parser.parse(&input);\n\t\tassert_eq!(output, Ok((\"ab\", 'c')));\n\t}\n\n\t#[test]\n\tfn recursive_parser() {\n\t\t#[derive(Debug, PartialEq)]\n\t\tenum Expr {\n\t\t\tEmpty,\n\t\t\tGroup(Box),\n\t\t}\n\t\tfn expr() -> Parser<'static, u8, Expr> {\n\t\t\t(sym(b'(') + call(expr) - sym(b')')).map(|(_, e)| Expr::Group(Box::new(e)))\n\t\t\t\t| empty().map(|_| Expr::Empty)\n\t\t}\n\t\tlet input = b\"(())\";\n\t\tlet parser = expr();\n\t\tlet output = parser.parse(input);\n\t\tassert_eq!(\n\t\t\toutput,\n\t\t\tOk(Expr::Group(Box::new(Expr::Group(Box::new(Expr::Empty)))))\n\t\t);\n\t}\n\n\t#[test]\n\tfn chain_parser() {\n\t\tlet input = b\"5oooooooo\";\n\t\t{\n\t\t\tlet parser = one_of(b\"0123456789\").map(|c| c - b'0')\n\t\t\t\t>> |n| take(n as usize) + sym(b'o').repeat(0..);\n\t\t\tassert_eq!(parser.parse(input), Ok((&b\"ooooo\"[..], vec![b'o'; 3])));\n\t\t}\n\t\t{\n\t\t\tlet parser =\n\t\t\t\tskip(1) * take(3) >> |v: &'static [u8]| take(v.len() + 2).map(move |u| (u, v));\n\t\t\tassert_eq!(parser.parse(input), Ok((&b\"ooooo\"[..], &b\"ooo\"[..])));\n\t\t}\n\t\t{\n\t\t\tlet parser = Parser::new(move |input, start| {\n\t\t\t\t(skip(1) * take(3))\n\t\t\t\t\t.parse_at(input, start)\n\t\t\t\t\t.and_then(|(v, pos)| {\n\t\t\t\t\t\ttake(v.len() + 2)\n\t\t\t\t\t\t\t.parse_at(input, pos)\n\t\t\t\t\t\t\t.map(|(u, end)| ((u, v), end))\n\t\t\t\t\t})\n\t\t\t});\n\t\t\tassert_eq!(parser.parse(input), Ok((&b\"ooooo\"[..], &b\"ooo\"[..])));\n\t\t}\n\t}\n\n\t#[test]\n\tfn repeat_at_least() {\n\t\tlet input = b\"xxxooo\";\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(1..2);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 1]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(1..);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(0..);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'y').repeat(0..);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'y').repeat(1..);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert!(output.is_err());\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(10..);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert!(output.is_err());\n\t\t}\n\t}\n\n\t#[test]\n\tfn repeat_up_to() {\n\t\tlet input = b\"xxxooo\";\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(..2);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 1]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(..4);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(..);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(..0);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(..10);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\t}\n\n\t#[test]\n\tfn repeat_up_to_inclusive() {\n\t\tlet input = b\"xxxooo\";\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(..=2);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 2]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(..=4);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(..=0);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(..=10);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\t}\n\n\t#[test]\n\tfn repeat_from_to_inclusive() {\n\t\tlet input = b\"xxxooo\";\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(1..=2);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 2]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(1..=4);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(0..=0);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(3..=10);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(4..=10);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert!(output.is_err())\n\t\t}\n\t}\n\n\t#[test]\n\tfn repeat_exactly() {\n\t\tlet input = b\"xxxooo\";\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(0);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(1);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 1]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(2);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 2]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(3);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(4);\n\t\t\tlet output = parser.parse(input);\n\t\t\tassert!(output.is_err())\n\t\t}\n\t}\n\n\t#[cfg(not(feature = \"trace\"))]\n\t#[test]\n\tfn named() {\n\t\tlet input = b\"xxxooo\";\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(3);\n\t\t\tlet output = parser.name(\"name_test_ok\").parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(4);\n\t\t\tlet output = parser.name(\"name_test_err\").parse(input);\n\t\t\tassert_eq!(\n\t\t\t\toutput,\n\t\t\t\tErr(Error::Custom {\n\t\t\t\t\tmessage: \"failed to parse name_test_err\".into(),\n\t\t\t\t\tposition: 0,\n\t\t\t\t\tinner: Some(Box::new(Error::Mismatch {\n\t\t\t\t\t\tmessage: \"expect repeat at least 4 times, found 3 times\".into(),\n\t\t\t\t\t\tposition: 0\n\t\t\t\t\t}))\n\t\t\t\t})\n\t\t\t)\n\t\t}\n\t}\n\n\t#[cfg(feature = \"trace\")]\n\t#[test]\n\t// Note: this doesn't test the tracing per se, just that the `name()` method executes\n\t// in the same way when the feature is turned on.\n\tfn named() {\n\t\tlet input = b\"xxxooo\";\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(3);\n\t\t\tlet output = parser.name(\"name_test_ok\").parse(input);\n\t\t\tassert_eq!(output, Ok(vec![b'x'; 3]))\n\t\t}\n\n\t\t{\n\t\t\tlet parser = sym(b'x').repeat(4);\n\t\t\tlet output = parser.name(\"name_test_err\").parse(input);\n\t\t\tassert_eq!(\n\t\t\t\toutput,\n\t\t\t\tErr(Error::Custom {\n\t\t\t\t\tmessage: \"failed to parse name_test_err\".into(),\n\t\t\t\t\tposition: 0,\n\t\t\t\t\tinner: Some(Box::new(Error::Mismatch {\n\t\t\t\t\t\tmessage: \"expect repeat at least 4 times, found 3 times\".into(),\n\t\t\t\t\t\tposition: 0\n\t\t\t\t\t}))\n\t\t\t\t})\n\t\t\t)\n\t\t}\n\t}\n}\n" }, { "path": "src/range.rs", "content": "use std::ops::{Bound, RangeBounds, Range, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive};\n\npub trait RangeArgument {\n\tfn start(&self) -> Bound<&usize>;\n\tfn end(&self) -> Bound<&usize>;\n}\n\nimpl RangeArgument for Range {\n\tfn start(&self) -> Bound<&usize> {\n self.start_bound()\n\t}\n\tfn end(&self) -> Bound<&usize> {\n self.end_bound()\n\t}\n}\n\nimpl RangeArgument for RangeFrom {\n\tfn start(&self) -> Bound<&usize> {\n self.start_bound()\n\t}\n\tfn end(&self) -> Bound<&usize> {\n self.end_bound()\n\t}\n}\n\nimpl RangeArgument for RangeFull {\n\tfn start(&self) -> Bound<&usize> {\n self.start_bound()\n\t}\n\tfn end(&self) -> Bound<&usize> {\n self.end_bound()\n\t}\n}\n\nimpl RangeArgument for RangeInclusive {\n fn start(&self) -> Bound<&usize> {\n self.start_bound()\n }\n fn end(&self) -> Bound<&usize> {\n self.end_bound()\n }\n}\n\nimpl RangeArgument for RangeTo {\n\tfn start(&self) -> Bound<&usize> {\n self.start_bound()\n\t}\n\tfn end(&self) -> Bound<&usize> {\n self.end_bound()\n\t}\n}\n\nimpl RangeArgument for RangeToInclusive {\n\tfn start(&self) -> Bound<&usize> {\n self.start_bound()\n\t}\n\tfn end(&self) -> Bound<&usize> {\n self.end_bound()\n\t}\n}\n\nimpl RangeArgument for usize { \n fn start(&self) -> Bound<&usize> {\n Bound::Included(self)\n }\n fn end(&self) -> Bound<&usize> {\n Bound::Included(self)\n }\n}\n\n#[cfg(test)]\nmod test {\n use super::*;\n\n fn accept(ra: R, expected: impl std::ops::RangeBounds)\n where\n R: RangeArgument,\n T: std::fmt::Debug + std::cmp::PartialEq {\n assert_eq!(ra.start(), expected.start_bound());\n assert_eq!(ra.end(), expected.end_bound());\n }\n\n #[test]\n fn unbounded() {\n accept::(.., ..)\n }\n\n #[test]\n fn up_to_inclusive() {\n accept::(..=2, ..=2)\n }\n\n #[test]\n fn up_to_exclusive() {\n accept::(..2, ..2)\n }\n\n #[test]\n fn from() {\n accept::(1.., 1..)\n }\n\n #[test]\n fn from_to_inclusive() {\n accept::(1..=2, 1..=2)\n }\n\n #[test]\n fn from_to_exclusive() {\n accept::(1..3, 1..3)\n }\n\n #[test]\n fn exactly() {\n accept::(42, 42..=42)\n }\n}\n" }, { "path": "src/result.rs", "content": "use std::{\n\terror,\n\tfmt::{self, Display},\n};\n\n/// Parser error.\n#[derive(Debug, PartialEq, Clone)]\npub enum Error {\n\tIncomplete,\n\tMismatch {\n\t\tmessage: String,\n\t\tposition: usize,\n\t},\n\tConversion {\n\t\tmessage: String,\n\t\tposition: usize,\n\t},\n\tExpect {\n\t\tmessage: String,\n\t\tposition: usize,\n\t\tinner: Box,\n\t},\n\tCustom {\n\t\tmessage: String,\n\t\tposition: usize,\n\t\tinner: Option>,\n\t},\n}\n\nimpl error::Error for Error {\n\tfn description(&self) -> &'static str {\n\t\t\"Parse error\"\n\t}\n}\n\nimpl Display for Error {\n\tfn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n\t\tmatch self {\n\t\t\tSelf::Incomplete => write!(f, \"Incomplete\"),\n\t\t\tSelf::Mismatch {\n\t\t\t\tref message,\n\t\t\t\tref position,\n\t\t\t} => write!(f, \"Mismatch at {}: {}\", position, message),\n\t\t\tSelf::Conversion {\n\t\t\t\tref message,\n\t\t\t\tref position,\n\t\t\t} => write!(f, \"Conversion failed at {}: {}\", position, message),\n\t\t\tSelf::Expect {\n\t\t\t\tref message,\n\t\t\t\tref position,\n\t\t\t\tref inner,\n\t\t\t} => write!(f, \"{} at {}: {}\", message, position, inner),\n\t\t\tSelf::Custom {\n\t\t\t\tref message,\n\t\t\t\tref position,\n\t\t\t\tinner: Some(ref inner),\n\t\t\t} => write!(f, \"{} at {}, (inner: {})\", message, position, inner),\n\t\t\tSelf::Custom {\n\t\t\t\tref message,\n\t\t\t\tref position,\n\t\t\t\tinner: None,\n\t\t\t} => write!(f, \"{} at {}\", message, position),\n\t\t}\n\t}\n}\n\n/// Parser result, `Result` ia alias of `Result`.\npub type Result = ::std::result::Result;\n" }, { "path": "src/set.rs", "content": "use std::{\n\tcmp::{PartialEq, PartialOrd},\n\tops::{Range, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive},\n\tstr,\n};\n\n/// Set relationship.\npub trait Set {\n\t/// Whether a set contains an element or not.\n\tfn contains(&self, elem: &T) -> bool;\n\n\t/// Convert to text for display.\n\tfn to_str(&self) -> &str {\n\t\t\"\"\n\t}\n}\n\nimpl Set for [T] {\n\tfn contains(&self, elem: &T) -> bool {\n\t\t(self as &[T]).contains(elem)\n\t}\n}\n\nimpl Set for str {\n\tfn contains(&self, elem: &char) -> bool {\n\t\t(self as &str).contains(*elem)\n\t}\n\n\tfn to_str(&self) -> &str {\n\t\tself\n\t}\n}\n\nimpl Set for Range {\n\tfn contains(&self, elem: &T) -> bool {\n\t\tself.start <= *elem && self.end > *elem\n\t}\n}\n\nimpl Set for RangeFrom {\n\tfn contains(&self, elem: &T) -> bool {\n\t\tself.start <= *elem\n\t}\n}\n\nimpl Set for RangeInclusive {\n\tfn contains(&self, elem: &T) -> bool {\n\t\tself.start() <= elem && self.end() >= elem\n\t}\n}\n\nimpl Set for RangeTo {\n\tfn contains(&self, elem: &T) -> bool {\n\t\tself.end > *elem\n\t}\n}\n\nimpl Set for RangeToInclusive {\n\tfn contains(&self, elem: &T) -> bool {\n\t\tself.end >= *elem\n\t}\n}\n\nimpl Set for RangeFull {\n\tfn contains(&self, _: &T) -> bool {\n\t\ttrue\n\t}\n\n\tfn to_str(&self) -> &str {\n\t\t\"..\"\n\t}\n}\n\nimpl Set for [u8; N] {\n\tfn contains(&self, elem: &u8) -> bool {\n\t\t(self as &[u8]).contains(elem)\n\t}\n\n\tfn to_str(&self) -> &str {\n\t\tstr::from_utf8(self).unwrap_or(\"\")\n\t}\n}\n\n#[cfg(test)]\nmod test {\n\tuse crate::parser::*;\n\n\t#[test]\n\tfn one_of_using_set() {\n\t\tassert!(one_of(b\"az\").parse(b\"a\").is_ok());\n\t\tassert!(one_of(b\"az\").parse(b\"1\").is_err());\n\t}\n\n\t#[test]\n\tfn one_of_using_range() {\n\t\tassert!(one_of(&(b'a'..b'z')).parse(b\"a\").is_ok());\n\t\tassert!(one_of(&(b'a'..b'z')).parse(b\"z\").is_err());\n\t\tassert!(one_of(&(b'a'..b'z')).parse(b\"1\").is_err());\n\t}\n\n\t#[test]\n\tfn one_of_using_range_to() {\n\t\tassert!(one_of(&(..b'z')).parse(b\"a\").is_ok());\n\t\tassert!(one_of(&(..b'z')).parse(b\"z\").is_err());\n\t\tassert!(one_of(&(..b'z')).parse(b\"1\").is_ok());\n\t}\n\n\t#[test]\n\tfn one_of_using_range_inclusive() {\n\t\tassert!(one_of(&(b'a'..=b'z')).parse(b\"a\").is_ok());\n\t\tassert!(one_of(&(b'a'..=b'z')).parse(b\"z\").is_ok());\n\t\tassert!(one_of(&(b'a'..=b'z')).parse(b\"1\").is_err());\n\t}\n\n\t#[test]\n\tfn one_of_using_range_to_inclusive() {\n\t\tassert!(one_of(&(..=b'z')).parse(b\"a\").is_ok());\n\t\tassert!(one_of(&(..=b'z')).parse(b\"z\").is_ok());\n\t\tassert!(one_of(&(..=b'z')).parse(b\"1\").is_ok());\n\t}\n\n\t#[test]\n\tfn one_of_using_full_range() {\n\t\tassert!(one_of(&(..)).parse(b\"a\").is_ok());\n\t\tassert!(one_of(&(..)).parse(b\"z\").is_ok());\n\t\tassert!(one_of(&(..)).parse(b\"1\").is_ok());\n\t}\n\n}" }, { "path": "src/utf8.rs", "content": "// Variants of parser functions specialized for matching UTF-8 strings and returning chars\n\nuse super::parser;\nuse super::{Error, Result};\nuse crate::range::RangeArgument;\nuse crate::set::Set;\nuse bstr::decode_utf8;\nuse std::fmt::Debug;\nuse std::ops::{Add, BitOr, Mul, Neg, Not, Shr, Sub};\nuse std::str;\n\n// / Parser combinator.\n//type Parse<'a, O> = dyn Fn(&'a [u8], usize) -> Result<(O, usize)> + 'a;\n\n/// Being wrapped in this struct guarantees that the parser within will only match valid UTF-8 strings.\npub struct Parser<'a, O>(parser::Parser<'a, u8, O>);\n\nimpl<'a, O> Parser<'a, O> {\n\t/// Create new parser.\n\tpub fn new

(parse: P) -> Self\n\twhere\n\t\tP: Fn(&'a [u8], usize) -> Result<(O, usize)> + 'a,\n\t{\n\t\tSelf(parser::Parser::new(parse))\n\t}\n\n\t/// Collect all matched input symbols.\n\t// This method is the primary reason utf8::Parser exists at all.\n\tpub fn collect(self) -> Parser<'a, &'a str>\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser(self.0.collect().map(\n\t\t\t// UNSAFE: Because we only could have constructed this object from other utf8::Parser objects, the match space must be valid UTF-8\n\t\t\t|s| unsafe { str::from_utf8_unchecked(s) },\n\t\t))\n\t}\n\n\t// Remaining methods in impl only delegate to base parser::Parser\n\n\t/// Apply the parser to parse input.\n\tpub fn parse(&self, input: &'a [u8]) -> Result {\n\t\tself.0.parse(input)\n\t}\n\n\t/// Parse input at specified byte position.\n\tpub fn parse_at(&self, input: &'a [u8], start: usize) -> Result<(O, usize)> {\n\t\tself.0.parse_at(input, start)\n\t}\n\n\t/// Apply the parser to parse input.\n\tpub fn parse_str(&self, input: &'a str) -> Result {\n\t\tself.0.parse(input.as_bytes())\n\t}\n\n\t/// Convert parser result to desired value.\n\tpub fn map(self, f: F) -> Parser<'a, U>\n\twhere\n\t\tF: Fn(O) -> U + 'a,\n\t\tO: 'a,\n\t\tU: 'a,\n\t{\n\t\tParser(self.0.map(f))\n\t}\n\n\t/// Convert parser result to desired value, fail in case of conversion error.\n\tpub fn convert(self, f: F) -> Parser<'a, U>\n\twhere\n\t\tF: Fn(O) -> ::std::result::Result + 'a,\n\t\tE: Debug,\n\t\tO: 'a,\n\t\tU: 'a,\n\t{\n\t\tParser(self.0.convert(f))\n\t}\n\n\t/// Cache parser output result to speed up backtracking.\n\tpub fn cache(self) -> Self\n\twhere\n\t\tO: Clone + 'a,\n\t{\n\t\tSelf(self.0.cache())\n\t}\n\n\t/// Get input position after matching parser.\n\tpub fn pos(self) -> Parser<'a, usize>\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser(self.0.pos())\n\t}\n\n\t/// Discard parser output.\n\tpub fn discard(self) -> Parser<'a, ()>\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser(self.0.discard())\n\t}\n\n\t/// Make parser optional.\n\tpub fn opt(self) -> Parser<'a, Option>\n\twhere\n\t\tO: 'a,\n\t{\n\t\tParser(self.0.opt())\n\t}\n\n\t/// `p.repeat(5)` repeat p exactly 5 times\n\t/// `p.repeat(0..)` repeat p zero or more times\n\t/// `p.repeat(1..)` repeat p one or more times\n\t/// `p.repeat(1..4)` match p at least 1 and at most 3 times\n\tpub fn repeat(self, range: R) -> Parser<'a, Vec>\n\twhere\n\t\tR: RangeArgument + Debug + 'a,\n\t\tO: 'a,\n\t{\n\t\tParser(self.0.repeat(range))\n\t}\n\n\t/// Give parser a name to identify parsing errors.\n\tpub fn name(self, name: &'a str) -> Self\n\twhere\n\t\tO: 'a,\n\t{\n\t\tSelf(self.0.name(name))\n\t}\n\n\t/// Mark parser as expected, abort early when failed in ordered choice.\n\tpub fn expect(self, name: &'a str) -> Self\n\twhere\n\t\tO: 'a,\n\t{\n\t\tSelf(self.0.expect(name))\n\t}\n}\n\nimpl<'a, O> From> for parser::Parser<'a, u8, O> {\n\tfn from(parser: Parser<'a, O>) -> Self {\n\t\tparser.0 // Simply unwrap\n\t}\n}\n\npub fn decode(slice: &[u8], start: usize) -> Result<(char, usize)> {\n\tlet (ch, size) = decode_utf8(&slice[start..]);\n\tlet Some(ch) = ch else {\n\t\treturn no_utf8(start, size);\n\t};\n\tOk((ch, size))\n}\n\n// Helper for functions that decode_utf8 and fail\nfn no_utf8(start: usize, size: usize) -> Result {\n\tErr(Error::Mismatch {\n\t\tmessage: if size == 0 {\n\t\t\t\"end of input reached\"\n\t\t} else {\n\t\t\t\"not UTF-8\"\n\t\t}\n\t\t.to_owned(),\n\t\tposition: start,\n\t})\n}\n\n/// Match any UTF-8 character.\npub fn any<'a>() -> Parser<'a, char> {\n\tParser::new(|input: &[u8], start: usize| {\n\t\tlet (ch, size) = decode(input, start)?;\n\t\tlet pos = start + size;\n\t\tOk((ch, pos))\n\t})\n}\n\n/// Match specific UTF-8 character.\npub fn sym<'a>(tag: char) -> Parser<'a, char> {\n\tParser::new(move |input: &[u8], start: usize| {\n\t\tlet (ch, size) = decode(input, start)?;\n\t\tif ch != tag {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"expect: {}, found: {}\", tag, ch),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tlet pos = start + size;\n\t\tOk((ch, pos))\n\t})\n}\n\n/// Success when sequence of chars matches current input.\npub fn seq<'a, 'b: 'a>(tag_str: &'b str) -> Parser<'a, &'a str> {\n\tlet tag = tag_str.as_bytes();\n\tParser::new(move |input: &'a [u8], start: usize| {\n\t\tlet mut index = 0;\n\t\tloop {\n\t\t\tlet pos = start + index;\n\t\t\tif index == tag.len() {\n\t\t\t\tlet result = &input[start..pos];\n\t\t\t\t// UNSAFE: Because slice is byte-identical to a str, it is known valid UTF-8\n\t\t\t\tlet result_str = unsafe { str::from_utf8_unchecked(result) };\n\t\t\t\treturn Ok((result_str, pos));\n\t\t\t}\n\t\t\tlet Some(s) = input.get(pos) else {\n\t\t\t\treturn Err(Error::Incomplete);\n\t\t\t};\n\t\t\tif tag[index] != *s {\n\t\t\t\treturn Err(Error::Mismatch {\n\t\t\t\t\tmessage: format!(\"seq {:?} at byte index: {}\", tag, pos),\n\t\t\t\t\tposition: pos,\n\t\t\t\t});\n\t\t\t}\n\t\t\tindex += 1;\n\t\t}\n\t})\n}\n\n/// Success when current input symbol is one of the set.\npub fn one_of<'a, S>(set: &'a S) -> Parser<'a, char>\nwhere\n\tS: Set + ?Sized,\n{\n\tParser::new(move |input: &'a [u8], start: usize| {\n\t\tlet (ch, size) = decode(input, start)?;\n\t\tif !set.contains(&ch) {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"expect one of: {}, found: {}\", set.to_str(), ch),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tlet pos = start + size;\n\t\tOk((ch, pos))\n\t})\n}\n\n/// Success when current input symbol is none of the set.\npub fn none_of<'a, S>(set: &'a S) -> Parser<'a, char>\nwhere\n\tS: Set + ?Sized,\n{\n\tParser::new(move |input: &'a [u8], start: usize| {\n\t\tlet (ch, size) = decode(input, start)?;\n\t\tif set.contains(&ch) {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"expect one of: {}, found: {}\", set.to_str(), ch),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tlet pos = start + size;\n\t\tOk((ch, pos))\n\t})\n}\n\n/// Success when predicate returns true on current input symbol.\npub fn is_a<'a, F>(predicate: F) -> Parser<'a, char>\nwhere\n\tF: Fn(char) -> bool + 'a,\n{\n\tParser::new(move |input: &'a [u8], start: usize| {\n\t\tlet (ch, size) = decode(input, start)?;\n\t\tif !predicate(ch) {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"is_a predicate failed on: {}\", ch),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tlet pos = start + size;\n\t\tOk((ch, pos))\n\t})\n}\n\n/// Success when predicate returns false on current input symbol.\npub fn not_a<'a, F>(predicate: F) -> Parser<'a, char>\nwhere\n\tF: Fn(char) -> bool + 'a,\n{\n\tParser::new(move |input: &'a [u8], start: usize| {\n\t\tlet (ch, size) = decode(input, start)?;\n\t\tif predicate(ch) {\n\t\t\treturn Err(Error::Mismatch {\n\t\t\t\tmessage: format!(\"is_a predicate failed on: {}\", ch),\n\t\t\t\tposition: start,\n\t\t\t});\n\t\t}\n\t\tlet pos = start + size;\n\t\tOk((ch, pos))\n\t})\n}\n\n/// Read n chars.\npub fn take<'a>(n: usize) -> Parser<'a, &'a str> {\n\tParser::new(move |input: &'a [u8], start: usize| {\n\t\tlet mut byte_pos = start;\n\t\tfor _ in 0..n {\n\t\t\tlet (ch, size) = decode_utf8(&input[start..]);\n\t\t\tif ch.is_none() {\n\t\t\t\treturn no_utf8(byte_pos, size);\n\t\t\t}\n\t\t\tbyte_pos += size;\n\t\t}\n\t\tlet result = &input[start..byte_pos];\n\t\t// UNSAFE: Because every char has been checked by decode_utf8, this string is known utf8\n\t\tlet result_str = unsafe { str::from_utf8_unchecked(result) };\n\t\tOk((result_str, byte_pos))\n\t})\n}\n\n/// Skip n symbols.\npub fn skip<'a>(n: usize) -> Parser<'a, ()> {\n\tParser::new(move |input: &'a [u8], start: usize| {\n\t\tlet mut byte_pos = start;\n\t\tfor _ in 0..n {\n\t\t\tlet (ch, size) = decode_utf8(&input[start..]);\n\t\t\tif ch.is_none() {\n\t\t\t\treturn no_utf8(byte_pos, size);\n\t\t\t}\n\t\t\tbyte_pos += size;\n\t\t}\n\t\tOk(((), byte_pos))\n\t})\n}\n\n/// Read n bytes exactly.\npub fn take_bytes<'a>(n: usize) -> Parser<'a, &'a str> {\n\tParser::new(move |input: &'a [u8], start: usize| {\n\t\t// FIXME: This runs in linear time because it checks each character.\n\t\t// If we could remember which inputs were passed in from parse_str() instead of parse(),\n\t\t// we could assume the characters are valid utf8 and run this in constant time by only checking\n\t\t// the final character using bstr::decode_last_utf8.\n\t\tlet mut byte_pos = start;\n\t\tloop {\n\t\t\tlet (ch, size) = decode_utf8(&input[start..]);\n\t\t\tif ch.is_none() {\n\t\t\t\treturn no_utf8(byte_pos, size);\n\t\t\t}\n\t\t\tbyte_pos += size;\n\t\t\tif byte_pos > n {\n\t\t\t\treturn Err(Error::Mismatch {\n\t\t\t\t\tmessage: \"range splits a UTF-8 character\".to_owned(),\n\t\t\t\t\tposition: start,\n\t\t\t\t});\n\t\t\t}\n\t\t\tif byte_pos == n {\n\t\t\t\tlet result = &input[start..byte_pos];\n\t\t\t\t// UNSAFE: Because every char has been checked by decode_utf8, this string is known utf8\n\t\t\t\tlet result_str = unsafe { str::from_utf8_unchecked(result) };\n\t\t\t\treturn Ok((result_str, byte_pos));\n\t\t\t}\n\t\t}\n\t})\n}\n\n/// Skip n bytes exactly.\npub fn skip_bytes<'a>(n: usize) -> Parser<'a, ()> {\n\tParser::new(move |input: &'a [u8], start: usize| {\n\t\t// FIXME: See note on take_bytes.\n\t\tlet mut byte_pos = start;\n\t\tloop {\n\t\t\tlet (ch, size) = decode_utf8(&input[start..]);\n\t\t\tif ch.is_none() {\n\t\t\t\treturn no_utf8(byte_pos, size);\n\t\t\t}\n\t\t\tbyte_pos += size;\n\t\t\tif byte_pos > n {\n\t\t\t\treturn Err(Error::Mismatch {\n\t\t\t\t\tmessage: \"range splits a UTF-8 character\".to_owned(),\n\t\t\t\t\tposition: start,\n\t\t\t\t});\n\t\t\t}\n\t\t\tif byte_pos == n {\n\t\t\t\treturn Ok(((), byte_pos));\n\t\t\t}\n\t\t}\n\t})\n}\n\n/// Chain two parsers where the second parser depends on the first's result.\nimpl<'a, O: 'a, U: 'a, F: Fn(O) -> Parser<'a, U> + 'a> Shr for Parser<'a, O> {\n\ttype Output = Parser<'a, U>;\n\n\tfn shr(self, other: F) -> Self::Output {\n\t\tParser::new(move |input: &'a [u8], start: usize| {\n\t\t\t(self.0.method)(input, start).and_then(|(out, pos)| (other(out).0.method)(input, pos))\n\t\t})\n\t}\n}\n\n// Note: There are no \"degrade to parser::Parser\" implementations for >>\n// because Rust cannot tell the difference between an FN(O)->U and an FN(O)->V.\n\n// Remaining functions in file only delegate to base parser::Parser\n\n/// Always succeeds, consume no input.\npub fn empty<'a>() -> Parser<'a, ()> {\n\tParser(parser::empty())\n}\n\n/// Parse separated list.\npub fn list<'a, O, U>(item: Parser<'a, O>, separator: Parser<'a, U>) -> Parser<'a, Vec>\nwhere\n\tO: 'a,\n\tU: 'a,\n{\n\tParser(parser::list(item.0, separator.0))\n}\n\n/// Call a parser factory, can be used to create recursive parsers.\npub fn call<'a, O, F>(parser_factory: F) -> Parser<'a, O>\nwhere\n\tO: 'a,\n\tF: Fn() -> Parser<'a, O> + 'a,\n{\n\tParser(parser::call(move || parser_factory().0))\n}\n\n/// Success when end of input is reached.\npub fn end<'a>() -> Parser<'a, ()> {\n\tParser(parser::end())\n}\n\n// And, Sub and Mul are similar enough we can implement them with macros\n\nmacro_rules! utf_op {\n ( $impl_name:ident, $fn_name:ident, $op:tt, $return_type:ty, $doc:expr ) => {\n \t#[doc=$doc]\n\t\timpl<'a, Left: 'a, Right: 'a> $impl_name> for Parser<'a, Left> {\n\t\t\ttype Output = Parser<'a, $return_type>;\n\n\t\t\tfn $fn_name (self, other: Parser<'a, Right>) -> Self::Output {\n\t\t\t\tParser(self.0 $op other.0)\n\t\t\t}\n\t\t}\n };\n}\n\nmacro_rules! utf_u8_op {\n ( $impl_name:ident, $fn_name:ident, $op:tt, $return_type:ty, $doc:expr ) => {\n \t#[doc=concat!($doc, \" (but degrade to non-utf8 parser)\")]\n\t\timpl<'a, Left: 'a, Right: 'a> $impl_name> for Parser<'a, Left> {\n\t\t\ttype Output = parser::Parser<'a, u8, $return_type>;\n\n\t\t\tfn $fn_name (self, other: parser::Parser<'a, u8, Right>) -> Self::Output {\n\t\t\t\tself.0 $op other\n\t\t\t}\n\t\t}\n };\n}\n\nmacro_rules! u8_utf_op {\n ( $impl_name:ident, $fn_name:ident, $op:tt, $return_type:ty, $doc:expr ) => {\n \t#[doc=concat!($doc, \" (but degrade to non-utf8 parser)\")]\n\t\timpl<'a, Left: 'a, Right: 'a> $impl_name> for parser::Parser<'a, u8, Left> {\n\t\t\ttype Output = parser::Parser<'a, u8, $return_type>;\n\n\t\t\tfn $fn_name (self, other: Parser<'a, Right>) -> Self::Output {\n\t\t\t\tself $op other.0\n\t\t\t}\n\t\t}\n };\n}\n\nmacro_rules! all_op {\n\t( $impl_name:ident, $fn_name:ident, $op:tt, $return_type:ty, $doc:expr ) => {\n\t\tutf_op!($impl_name, $fn_name, $op, $return_type, $doc);\n\t\tutf_u8_op!($impl_name, $fn_name, $op, $return_type, $doc);\n\t\tu8_utf_op!($impl_name, $fn_name, $op, $return_type, $doc);\n\t};\n}\n\nall_op!(Add, add, +, (Left, Right), \"Sequence reserve value\");\n\nall_op!(Sub, sub, -, Left, \"Sequence discard second value\");\n\nall_op!(Mul, mul, *, Right, \"Sequence discard first value\");\n\n/// Ordered choice\nimpl<'a, O: 'a> BitOr for Parser<'a, O> {\n\ttype Output = Self;\n\n\tfn bitor(self, other: Self) -> Self {\n\t\tSelf(self.0 | other.0)\n\t}\n}\n\n/// Ordered choice (but degrade to non-utf8 parser)\nimpl<'a, O: 'a> BitOr> for Parser<'a, O> {\n\ttype Output = parser::Parser<'a, u8, O>;\n\n\tfn bitor(self, other: parser::Parser<'a, u8, O>) -> Self::Output {\n\t\tself.0 | other\n\t}\n}\n\n/// Ordered choice (but degrade to non-utf8 parser)\nimpl<'a, O: 'a> BitOr> for parser::Parser<'a, u8, O> {\n\ttype Output = parser::Parser<'a, u8, O>;\n\n\tfn bitor(self, other: Parser<'a, O>) -> Self::Output {\n\t\tself | other.0\n\t}\n}\n\n/// And predicate\nimpl<'a, O: 'a> Neg for Parser<'a, O> {\n\ttype Output = Parser<'a, bool>;\n\n\tfn neg(self) -> Self::Output {\n\t\tParser(-self.0)\n\t}\n}\n\n/// Not predicate\nimpl<'a, O: 'a> Not for Parser<'a, O> {\n\ttype Output = Parser<'a, bool>;\n\n\tfn not(self) -> Self::Output {\n\t\tParser(!self.0)\n\t}\n}\n" }, { "path": "tests/list.rs", "content": "extern crate pom;\n\nuse pom::parser::*;\nuse pom::Parser;\n\nfn spaces() -> Parser {\n\tone_of(b\" \").repeat(1..).discard()\n}\n\nfn works() -> Parser> {\n\tlist(one_of(b\"abc\"), spaces() * seq(b\"and\") - spaces())\n}\n\nfn dangle() -> Parser, &'static [u8])> {\n\tlist(one_of(b\"abc\"), spaces() * seq(b\"and\") - spaces()) + seq(b\" and\")\n}\n\n#[test]\nfn test_list() {\n\tlet one = b\"a and b and c\";\n\tassert_eq!(works().parse(one), Ok(vec![b'a', b'b', b'c']));\n\n\tlet two = b\"a and b and c and \";\n\tassert_eq!(\n\t\tdangle().parse(two),\n\t\tOk((vec![b'a', b'b', b'c'], &b\" and\"[..]))\n\t);\n}\n" } ]