Repository: georgewfraser/vscode-tree-sitter Branch: master Commit: 471169a992a8 Files: 57 Total size: 867.8 KB Directory structure: gitextract_dzd0zkcs/ ├── .gitignore ├── .vscode/ │ ├── launch.json │ ├── settings.json │ └── tasks.json ├── .vscodeignore ├── LICENSE.md ├── README.md ├── TODO.md ├── azure-pipelines.yml ├── examples/ │ ├── cpp/ │ │ ├── marker-index.h │ │ └── rule.cc │ ├── go/ │ │ ├── letter_test.go │ │ ├── no_newline_at_eof.go │ │ ├── proc.go │ │ ├── small.go │ │ ├── type_switch.go │ │ └── value.go │ ├── javascript/ │ │ ├── destructuring.js │ │ ├── expressions.js │ │ ├── literals.js │ │ ├── semicolon_insertion.js │ │ └── statements.js │ ├── ruby/ │ │ ├── classes.rb │ │ ├── comments.rb │ │ ├── control-flow.rb │ │ ├── declarations.rb │ │ ├── expressions.rb │ │ ├── literals.rb │ │ └── statements.rb │ ├── rust/ │ │ ├── ast.rs │ │ ├── keywords.txt │ │ └── scratch.rs │ └── typescript/ │ ├── keywords.txt │ ├── parser.ts │ └── small.ts ├── package.json ├── parsers/ │ ├── tree-sitter-cpp.wasm │ ├── tree-sitter-go.wasm │ ├── tree-sitter-javascript.wasm │ ├── tree-sitter-ruby.wasm │ ├── tree-sitter-rust.wasm │ └── tree-sitter-typescript.wasm ├── scripts/ │ ├── build.sh │ └── gen-parsers.sh ├── src/ │ ├── benchmark.ts │ ├── colors.ts │ ├── extension.ts │ ├── print.ts │ ├── scopes.ts │ └── test.ts ├── textmate/ │ ├── cpp.tmLanguage.json │ ├── go.tmLanguage.json │ ├── ruby.tmLanguage.json │ ├── rust.tmLanguage.json │ └── typescript.tmLanguage.json ├── tsconfig.json └── tslint.json ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ out node_modules .vscode-test/ *.vsix *.bin ================================================ FILE: .vscode/launch.json ================================================ // A launch configuration that compiles the extension and then opens it inside a new window // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 { "version": "0.2.0", "configurations": [{ "name": "Run Extension", "type": "extensionHost", "request": "launch", "runtimeExecutable": "${execPath}", "args": [ "--extensionDevelopmentPath=${workspaceFolder}" ], "outFiles": [ "${workspaceFolder}/out/**/*.js" ], "preLaunchTask": "npm: compile" }, { "name": "Extension Tests", "type": "extensionHost", "request": "launch", "runtimeExecutable": "${execPath}", "args": [ "--extensionDevelopmentPath=${workspaceFolder}", "--extensionTestsPath=${workspaceFolder}/out/test" ], "outFiles": [ "${workspaceFolder}/out/test/**/*.js" ], "preLaunchTask": "npm: compile" }, { "name": "Debug tests", "type": "node", "request": "launch", "cwd": "${workspaceFolder}", "runtimeExecutable": "npm", "runtimeArgs": [ "run-script", "debug" ], "port": 9229 } ] } ================================================ FILE: .vscode/settings.json ================================================ // Place your settings in this file to overwrite default and user settings. { "files.exclude": { "out": false // set this to true to hide the "out" folder with the compiled JS files }, "search.exclude": { "out": true // set this to false to include "out" folder in search results }, // Turn off tsc task auto detection since we have the necessary tasks as npm scripts "typescript.tsc.autoDetect": "off" } ================================================ FILE: .vscode/tasks.json ================================================ // See https://go.microsoft.com/fwlink/?LinkId=733558 // for the documentation about the tasks.json format { "version": "2.0.0", "tasks": [ { "type": "npm", "script": "watch", "problemMatcher": "$tsc-watch", "isBackground": true, "presentation": { "reveal": "never" }, "group": { "kind": "build", "isDefault": true } } ] } ================================================ FILE: .vscodeignore ================================================ .vscode/** .vscode-test/** out/test/** src/** .gitignore vsc-extension-quickstart.md **/tsconfig.json **/tslint.json **/*.map **/*.ts examples/** ================================================ FILE: LICENSE.md ================================================ The MIT License (MIT) Copyright (c) 2016 George Fraser Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # Tree Sitter for VSCode [Deprecated] **With the improving support for custom syntax coloring through language server, this extension is no longer needed** This extension gives VSCode support for [tree-sitter](http://tree-sitter.github.io/tree-sitter/) syntax coloring. Examples with tree-sitter coloring on the right: ## Go ![Go](./screenshots/go.png) ## Rust ![Rust](./screenshots/rust.png) ## C++ ![C++](./screenshots/cpp.png) ## Ruby ![Ruby](./screenshots/ruby.png) ## Javascript / Typescript ![Typescript](./screenshots/typescript.png) ## Contributing ### Fixing colorization of an existing language If you see something getting colored wrong, or something that should be colored but isn't, you can help! The simplest way to help is to create an issue with a simple example, a screenshot, and an explanation of what is wrong. You are also welcome to fix the problem yourself and submit a PR. Colorization is performed by the various `colorLanguage(x, editor)` functions in `src/colors.ts`. When working on the colorization rules, please keep in mind two core principles: 1. Good colorization is *consistent*. It's better to not color at all than to color inconsistently. 2. Good colorization is *selective*. The fewer things that we color, the more emphasis the color gives. ### Adding a new language It's straightforward to add any [language with a tree-sitter grammar](https://tree-sitter.github.io/tree-sitter/). 1. Add a dependency on the npm package for that language: `npm install tree-sitter-yourlang`. 2. Add a color function to `./src/colors.ts` 3. Add a language to the dictionary at the top of `./src/extension.ts` 4. Add a **simplified** TextMate grammar to `./textmate/yourlang.tmLanguage.json`. The job of this textmate grammar is just to color keywords and simple literals; anything tricky should be left white and colored by your color function. 5. Add a reference to the grammar to the [contributes.grammars section of package.json](https://github.com/georgewfraser/vscode-tree-sitter/blob/fb4400b78481845c6a8497d079508d28aea25c19/package.json#L26). `yourlang` must be a [VSCode language identifier](https://code.visualstudio.com/docs/languages/identifiers). 6. Add a reference to `onLanguage:yourlang` to the [activationEvents section of package.json](https://github.com/georgewfraser/vscode-tree-sitter/blob/fb4400b78481845c6a8497d079508d28aea25c19/package.json#L18). `yourlang` must be a [VSCode language identifier](https://code.visualstudio.com/docs/languages/identifiers). 7. Add an example to `examples/yourlang`. 8. Hit `F5` in VSCode, with this project open, to test your changes. 9. Take a screenshot comparing before-and-after and add it to the above list. 10. Submit a PR! ================================================ FILE: TODO.md ================================================ ## Bugs - Tree-sitter scope colors are wrong while the user is previewing other themes - Put back react support for .js and .tsx ## Features - Folding-range provider https://code.visualstudio.com/api/references/vscode-api#FoldingRangeProvider - Extend-selection provider https://code.visualstudio.com/api/references/vscode-api#SelectionRangeProvider - Document highlight provider https://code.visualstudio.com/api/references/vscode-api#DocumentHighlightProvider ================================================ FILE: azure-pipelines.yml ================================================ # Node.js # Build a general Node.js project with npm. # Add steps that analyze code, save build artifacts, deploy, and more: # https://docs.microsoft.com/azure/devops/pipelines/languages/javascript trigger: - master pool: vmImage: ubuntu-16.04 steps: - task: NodeTool@0 inputs: versionSpec: '10.x' displayName: 'Install Node.js' - script: 'npm install' displayName: 'Install NPM deps' - script: 'npm run compile' displayName: 'Compile Typescript' - script: 'node out/test.js' displayName: 'Run tests' failOnStderr: true ================================================ FILE: examples/cpp/marker-index.h ================================================ #ifndef MARKER_INDEX_H_ #define MARKER_INDEX_H_ #include #include #include "flat_set.h" #include "point.h" #include "range.h" class MarkerIndex { public: using MarkerId = unsigned; using MarkerIdSet = flat_set; struct SpliceResult { flat_set touch; flat_set inside; flat_set overlap; flat_set surround; }; struct Boundary { Point position; flat_set starting; flat_set ending; }; struct BoundaryQueryResult { std::vector containing_start; std::vector boundaries; }; MarkerIndex(unsigned seed = 0u); ~MarkerIndex(); int generate_random_number(); void insert(MarkerId id, Point start, Point end); void set_exclusive(MarkerId id, bool exclusive); void remove(MarkerId id); bool has(MarkerId id); SpliceResult splice(Point start, Point old_extent, Point new_extent); Point get_start(MarkerId id) const; Point get_end(MarkerId id) const; Range get_range(MarkerId id) const; int compare(MarkerId id1, MarkerId id2) const; flat_set find_intersecting(Point start, Point end); flat_set find_containing(Point start, Point end); flat_set find_contained_in(Point start, Point end); flat_set find_starting_in(Point start, Point end); flat_set find_starting_at(Point position); flat_set find_ending_in(Point start, Point end); flat_set find_ending_at(Point position); BoundaryQueryResult find_boundaries_after(Point start, size_t max_count); std::unordered_map dump(); private: friend class Iterator; struct Node { Node *parent; Node *left; Node *right; Point left_extent; flat_set left_marker_ids; flat_set right_marker_ids; flat_set start_marker_ids; flat_set end_marker_ids; int priority; Node(Node *parent, Point left_extent); bool is_marker_endpoint(); }; class Iterator { public: Iterator(MarkerIndex *marker_index); void reset(); Node* insert_marker_start(const MarkerId &id, const Point &start_position, const Point &end_position); Node* insert_marker_end(const MarkerId &id, const Point &start_position, const Point &end_position); Node* insert_splice_boundary(const Point &position, bool is_insertion_end); void find_intersecting(const Point &start, const Point &end, flat_set *result); void find_contained_in(const Point &start, const Point &end, flat_set *result); void find_starting_in(const Point &start, const Point &end, flat_set *result); void find_ending_in(const Point &start, const Point &end, flat_set *result); void find_boundaries_after(Point start, size_t max_count, BoundaryQueryResult *result); std::unordered_map dump(); private: void ascend(); void descend_left(); void descend_right(); void move_to_successor(); void seek_to_first_node_greater_than_or_equal_to(const Point &position); void mark_right(const MarkerId &id, const Point &start_position, const Point &end_position); void mark_left(const MarkerId &id, const Point &start_position, const Point &end_position); Node* insert_left_child(const Point &position); Node* insert_right_child(const Point &position); void check_intersection(const Point &start, const Point &end, flat_set *results); void cache_node_position() const; MarkerIndex *marker_index; Node *current_node; Point current_node_position; Point left_ancestor_position; Point right_ancestor_position; std::vector left_ancestor_position_stack; std::vector right_ancestor_position_stack; }; Point get_node_position(const Node *node) const; void delete_node(Node *node); void delete_subtree(Node *node); void bubble_node_up(Node *node); void bubble_node_down(Node *node); void rotate_node_left(Node *pivot); void rotate_node_right(Node *pivot); void get_starting_and_ending_markers_within_subtree(const Node *node, flat_set *starting, flat_set *ending); void populate_splice_invalidation_sets(SpliceResult *invalidated, const Node *start_node, const Node *end_node, const flat_set &starting_inside_splice, const flat_set &ending_inside_splice); std::default_random_engine random_engine; std::uniform_int_distribution random_distribution; Node *root; std::unordered_map start_nodes_by_id; std::unordered_map end_nodes_by_id; Iterator iterator; flat_set exclusive_marker_ids; mutable std::unordered_map node_position_cache; }; #endif // MARKER_INDEX_H_ ================================================ FILE: examples/cpp/rule.cc ================================================ #include "compiler/rule.h" #include "compiler/util/hash_combine.h" namespace tree_sitter { namespace rules { using std::move; using std::vector; using util::hash_combine; Rule::Rule(const Rule &other) : blank_(Blank{}), type(BlankType) { *this = other; } Rule::Rule(Rule &&other) noexcept : blank_(Blank{}), type(BlankType) { *this = move(other); } static void destroy_value(Rule *rule) { switch (rule->type) { case Rule::BlankType: return rule->blank_.~Blank(); case Rule::CharacterSetType: return rule->character_set_.~CharacterSet(); case Rule::StringType: return rule->string_ .~String(); case Rule::PatternType: return rule->pattern_ .~Pattern(); case Rule::NamedSymbolType: return rule->named_symbol_.~NamedSymbol(); case Rule::SymbolType: return rule->symbol_ .~Symbol(); case Rule::ChoiceType: return rule->choice_ .~Choice(); case Rule::MetadataType: return rule->metadata_ .~Metadata(); case Rule::RepeatType: return rule->repeat_ .~Repeat(); case Rule::SeqType: return rule->seq_ .~Seq(); } } Rule &Rule::operator=(const Rule &other) { destroy_value(this); type = other.type; switch (type) { case BlankType: new (&blank_) Blank(other.blank_); break; case CharacterSetType: new (&character_set_) CharacterSet(other.character_set_); break; case StringType: new (&string_) String(other.string_); break; case PatternType: new (&pattern_) Pattern(other.pattern_); break; case NamedSymbolType: new (&named_symbol_) NamedSymbol(other.named_symbol_); break; case SymbolType: new (&symbol_) Symbol(other.symbol_); break; case ChoiceType: new (&choice_) Choice(other.choice_); break; case MetadataType: new (&metadata_) Metadata(other.metadata_); break; case RepeatType: new (&repeat_) Repeat(other.repeat_); break; case SeqType: new (&seq_) Seq(other.seq_); break; } return *this; } Rule &Rule::operator=(Rule &&other) noexcept { destroy_value(this); type = other.type; switch (type) { case BlankType: new (&blank_) Blank(move(other.blank_)); break; case CharacterSetType: new (&character_set_) CharacterSet(move(other.character_set_)); break; case StringType: new (&string_) String(move(other.string_)); break; case PatternType: new (&pattern_) Pattern(move(other.pattern_)); break; case NamedSymbolType: new (&named_symbol_) NamedSymbol(move(other.named_symbol_)); break; case SymbolType: new (&symbol_) Symbol(move(other.symbol_)); break; case ChoiceType: new (&choice_) Choice(move(other.choice_)); break; case MetadataType: new (&metadata_) Metadata(move(other.metadata_)); break; case RepeatType: new (&repeat_) Repeat(move(other.repeat_)); break; case SeqType: new (&seq_) Seq(move(other.seq_)); break; } other.type = BlankType; other.blank_ = Blank{}; return *this; } Rule::~Rule() noexcept { destroy_value(this); } bool Rule::operator==(const Rule &other) const { if (type != other.type) return false; switch (type) { case Rule::CharacterSetType: return character_set_ == other.character_set_; case Rule::StringType: return string_ == other.string_; case Rule::PatternType: return pattern_ == other.pattern_; case Rule::NamedSymbolType: return named_symbol_ == other.named_symbol_; case Rule::SymbolType: return symbol_ == other.symbol_; case Rule::ChoiceType: return choice_ == other.choice_; case Rule::MetadataType: return metadata_ == other.metadata_; case Rule::RepeatType: return repeat_ == other.repeat_; case Rule::SeqType: return seq_ == other.seq_; default: return blank_ == other.blank_; } } template <> bool Rule::is() const { return type == BlankType; } template <> bool Rule::is() const { return type == SymbolType; } template <> bool Rule::is() const { return type == RepeatType; } template <> const Symbol & Rule::get_unchecked() const { return symbol_; } static inline void add_choice_element(std::vector *elements, const Rule &new_rule) { new_rule.match( [elements](Choice choice) { for (auto &element : choice.elements) { add_choice_element(elements, element); } }, [elements](auto rule) { for (auto &element : *elements) { if (element == rule) return; } elements->push_back(rule); } ); } Rule Rule::choice(const vector &rules) { vector elements; for (auto &element : rules) { add_choice_element(&elements, element); } return (elements.size() == 1) ? elements.front() : Choice{elements}; } Rule Rule::repeat(const Rule &rule) { return rule.is() ? rule : Repeat{rule}; } Rule Rule::seq(const vector &rules) { Rule result; for (const auto &rule : rules) { rule.match( [](Blank) {}, [&](Metadata metadata) { if (!metadata.rule->is()) { result = Seq{result, rule}; } }, [&](auto) { if (result.is()) { result = rule; } else { result = Seq{result, rule}; } } ); } return result; } } // namespace rules } // namespace tree_sitter namespace std { size_t hash::operator()(const Symbol &symbol) const { auto result = hash()(symbol.index); hash_combine(&result, hash()(symbol.type)); return result; } size_t hash::operator()(const NamedSymbol &symbol) const { return hash()(symbol.value); } size_t hash::operator()(const Pattern &symbol) const { return hash()(symbol.value); } size_t hash::operator()(const String &symbol) const { return hash()(symbol.value); } size_t hash::operator()(const CharacterSet &character_set) const { size_t result = 0; hash_combine(&result, character_set.includes_all); hash_combine(&result, character_set.included_chars.size()); for (uint32_t c : character_set.included_chars) { hash_combine(&result, c); } hash_combine(&result, character_set.excluded_chars.size()); for (uint32_t c : character_set.excluded_chars) { hash_combine(&result, c); } return result; } size_t hash::operator()(const Blank &blank) const { return 0; } size_t hash::operator()(const Choice &choice) const { size_t result = 0; for (const auto &element : choice.elements) { symmetric_hash_combine(&result, element); } return result; } size_t hash::operator()(const Repeat &repeat) const { size_t result = 0; hash_combine(&result, *repeat.rule); return result; } size_t hash::operator()(const Seq &seq) const { size_t result = 0; hash_combine(&result, *seq.left); hash_combine(&result, *seq.right); return result; } size_t hash::operator()(const Metadata &metadata) const { size_t result = 0; hash_combine(&result, *metadata.rule); hash_combine(&result, metadata.params.precedence); hash_combine(&result, metadata.params.associativity); hash_combine(&result, metadata.params.has_precedence); hash_combine(&result, metadata.params.has_associativity); hash_combine(&result, metadata.params.is_token); hash_combine(&result, metadata.params.is_string); hash_combine(&result, metadata.params.is_active); hash_combine(&result, metadata.params.is_main_token); return result; } size_t hash::operator()(const Rule &rule) const { size_t result = hash()(rule.type); switch (rule.type) { case Rule::CharacterSetType: return result ^ hash()(rule.character_set_); case Rule::StringType: return result ^ hash()(rule.string_); case Rule::PatternType: return result ^ hash()(rule.pattern_); case Rule::NamedSymbolType: return result ^ hash()(rule.named_symbol_); case Rule::SymbolType: return result ^ hash()(rule.symbol_); case Rule::ChoiceType: return result ^ hash()(rule.choice_); case Rule::MetadataType: return result ^ hash()(rule.metadata_); case Rule::RepeatType: return result ^ hash()(rule.repeat_); case Rule::SeqType: return result ^ hash()(rule.seq_); default: return result ^ hash()(rule.blank_); } } } // namespace std ================================================ FILE: examples/go/letter_test.go ================================================ // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package unicode_test import ( "flag" "fmt" "runtime" "sort" "testing" . "unicode" ) var upperTest = []rune{ 0x41, 0xc0, 0xd8, 0x100, 0x139, 0x14a, 0x178, 0x181, 0x376, 0x3cf, 0x13bd, 0x1f2a, 0x2102, 0x2c00, 0x2c10, 0x2c20, 0xa650, 0xa722, 0xff3a, 0x10400, 0x1d400, 0x1d7ca, } var notupperTest = []rune{ 0x40, 0x5b, 0x61, 0x185, 0x1b0, 0x377, 0x387, 0x2150, 0xab7d, 0xffff, 0x10000, } var letterTest = []rune{ 0x41, 0x61, 0xaa, 0xba, 0xc8, 0xdb, 0xf9, 0x2ec, 0x535, 0x620, 0x6e6, 0x93d, 0xa15, 0xb99, 0xdc0, 0xedd, 0x1000, 0x1200, 0x1312, 0x1401, 0x1885, 0x2c00, 0xa800, 0xf900, 0xfa30, 0xffda, 0xffdc, 0x10000, 0x10300, 0x10400, 0x20000, 0x2f800, 0x2fa1d, } var notletterTest = []rune{ 0x20, 0x35, 0x375, 0x619, 0x700, 0xfffe, 0x1ffff, 0x10ffff, } // Contains all the special cased Latin-1 chars. var spaceTest = []rune{ 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x20, 0x85, 0xA0, 0x2000, 0x3000, } type caseT struct { cas int in, out rune } var caseTest = []caseT{ // errors {-1, '\n', 0xFFFD}, {UpperCase, -1, -1}, {UpperCase, 1 << 30, 1 << 30}, // ASCII (special-cased so test carefully) {UpperCase, '\n', '\n'}, {UpperCase, 'a', 'A'}, {UpperCase, 'A', 'A'}, {UpperCase, '7', '7'}, {LowerCase, '\n', '\n'}, {LowerCase, 'a', 'a'}, {LowerCase, 'A', 'a'}, {LowerCase, '7', '7'}, {TitleCase, '\n', '\n'}, {TitleCase, 'a', 'A'}, {TitleCase, 'A', 'A'}, {TitleCase, '7', '7'}, // Latin-1: easy to read the tests! {UpperCase, 0x80, 0x80}, {UpperCase, 'Å', 'Å'}, {UpperCase, 'å', 'Å'}, {LowerCase, 0x80, 0x80}, {LowerCase, 'Å', 'å'}, {LowerCase, 'å', 'å'}, {TitleCase, 0x80, 0x80}, {TitleCase, 'Å', 'Å'}, {TitleCase, 'å', 'Å'}, // 0131;LATIN SMALL LETTER DOTLESS I;Ll;0;L;;;;;N;;;0049;;0049 {UpperCase, 0x0131, 'I'}, {LowerCase, 0x0131, 0x0131}, {TitleCase, 0x0131, 'I'}, // 0133;LATIN SMALL LIGATURE IJ;Ll;0;L; 0069 006A;;;;N;LATIN SMALL LETTER I J;;0132;;0132 {UpperCase, 0x0133, 0x0132}, {LowerCase, 0x0133, 0x0133}, {TitleCase, 0x0133, 0x0132}, // 212A;KELVIN SIGN;Lu;0;L;004B;;;;N;DEGREES KELVIN;;;006B; {UpperCase, 0x212A, 0x212A}, {LowerCase, 0x212A, 'k'}, {TitleCase, 0x212A, 0x212A}, // From an UpperLower sequence // A640;CYRILLIC CAPITAL LETTER ZEMLYA;Lu;0;L;;;;;N;;;;A641; {UpperCase, 0xA640, 0xA640}, {LowerCase, 0xA640, 0xA641}, {TitleCase, 0xA640, 0xA640}, // A641;CYRILLIC SMALL LETTER ZEMLYA;Ll;0;L;;;;;N;;;A640;;A640 {UpperCase, 0xA641, 0xA640}, {LowerCase, 0xA641, 0xA641}, {TitleCase, 0xA641, 0xA640}, // A64E;CYRILLIC CAPITAL LETTER NEUTRAL YER;Lu;0;L;;;;;N;;;;A64F; {UpperCase, 0xA64E, 0xA64E}, {LowerCase, 0xA64E, 0xA64F}, {TitleCase, 0xA64E, 0xA64E}, // A65F;CYRILLIC SMALL LETTER YN;Ll;0;L;;;;;N;;;A65E;;A65E {UpperCase, 0xA65F, 0xA65E}, {LowerCase, 0xA65F, 0xA65F}, {TitleCase, 0xA65F, 0xA65E}, // From another UpperLower sequence // 0139;LATIN CAPITAL LETTER L WITH ACUTE;Lu;0;L;004C 0301;;;;N;LATIN CAPITAL LETTER L ACUTE;;;013A; {UpperCase, 0x0139, 0x0139}, {LowerCase, 0x0139, 0x013A}, {TitleCase, 0x0139, 0x0139}, // 013F;LATIN CAPITAL LETTER L WITH MIDDLE DOT;Lu;0;L; 004C 00B7;;;;N;;;;0140; {UpperCase, 0x013f, 0x013f}, {LowerCase, 0x013f, 0x0140}, {TitleCase, 0x013f, 0x013f}, // 0148;LATIN SMALL LETTER N WITH CARON;Ll;0;L;006E 030C;;;;N;LATIN SMALL LETTER N HACEK;;0147;;0147 {UpperCase, 0x0148, 0x0147}, {LowerCase, 0x0148, 0x0148}, {TitleCase, 0x0148, 0x0147}, // Lowercase lower than uppercase. // AB78;CHEROKEE SMALL LETTER GE;Ll;0;L;;;;;N;;;13A8;;13A8 {UpperCase, 0xab78, 0x13a8}, {LowerCase, 0xab78, 0xab78}, {TitleCase, 0xab78, 0x13a8}, {UpperCase, 0x13a8, 0x13a8}, {LowerCase, 0x13a8, 0xab78}, {TitleCase, 0x13a8, 0x13a8}, // Last block in the 5.1.0 table // 10400;DESERET CAPITAL LETTER LONG I;Lu;0;L;;;;;N;;;;10428; {UpperCase, 0x10400, 0x10400}, {LowerCase, 0x10400, 0x10428}, {TitleCase, 0x10400, 0x10400}, // 10427;DESERET CAPITAL LETTER EW;Lu;0;L;;;;;N;;;;1044F; {UpperCase, 0x10427, 0x10427}, {LowerCase, 0x10427, 0x1044F}, {TitleCase, 0x10427, 0x10427}, // 10428;DESERET SMALL LETTER LONG I;Ll;0;L;;;;;N;;;10400;;10400 {UpperCase, 0x10428, 0x10400}, {LowerCase, 0x10428, 0x10428}, {TitleCase, 0x10428, 0x10400}, // 1044F;DESERET SMALL LETTER EW;Ll;0;L;;;;;N;;;10427;;10427 {UpperCase, 0x1044F, 0x10427}, {LowerCase, 0x1044F, 0x1044F}, {TitleCase, 0x1044F, 0x10427}, // First one not in the 5.1.0 table // 10450;SHAVIAN LETTER PEEP;Lo;0;L;;;;;N;;;;; {UpperCase, 0x10450, 0x10450}, {LowerCase, 0x10450, 0x10450}, {TitleCase, 0x10450, 0x10450}, // Non-letters with case. {LowerCase, 0x2161, 0x2171}, {UpperCase, 0x0345, 0x0399}, } func TestIsLetter(t *testing.T) { for _, r := range upperTest { if !IsLetter(r) { t.Errorf("IsLetter(U+%04X) = false, want true", r) } } for _, r := range letterTest { if !IsLetter(r) { t.Errorf("IsLetter(U+%04X) = false, want true", r) } } for _, r := range notletterTest { if IsLetter(r) { t.Errorf("IsLetter(U+%04X) = true, want false", r) } } } func TestIsUpper(t *testing.T) { for _, r := range upperTest { if !IsUpper(r) { t.Errorf("IsUpper(U+%04X) = false, want true", r) } } for _, r := range notupperTest { if IsUpper(r) { t.Errorf("IsUpper(U+%04X) = true, want false", r) } } for _, r := range notletterTest { if IsUpper(r) { t.Errorf("IsUpper(U+%04X) = true, want false", r) } } } func caseString(c int) string { switch c { case UpperCase: return "UpperCase" case LowerCase: return "LowerCase" case TitleCase: return "TitleCase" } return "ErrorCase" } func TestTo(t *testing.T) { for _, c := range caseTest { r := To(c.cas, c.in) if c.out != r { t.Errorf("To(U+%04X, %s) = U+%04X want U+%04X", c.in, caseString(c.cas), r, c.out) } } } func TestToUpperCase(t *testing.T) { for _, c := range caseTest { if c.cas != UpperCase { continue } r := ToUpper(c.in) if c.out != r { t.Errorf("ToUpper(U+%04X) = U+%04X want U+%04X", c.in, r, c.out) } } } func TestToLowerCase(t *testing.T) { for _, c := range caseTest { if c.cas != LowerCase { continue } r := ToLower(c.in) if c.out != r { t.Errorf("ToLower(U+%04X) = U+%04X want U+%04X", c.in, r, c.out) } } } func TestToTitleCase(t *testing.T) { for _, c := range caseTest { if c.cas != TitleCase { continue } r := ToTitle(c.in) if c.out != r { t.Errorf("ToTitle(U+%04X) = U+%04X want U+%04X", c.in, r, c.out) } } } func TestIsSpace(t *testing.T) { for _, c := range spaceTest { if !IsSpace(c) { t.Errorf("IsSpace(U+%04X) = false; want true", c) } } for _, c := range letterTest { if IsSpace(c) { t.Errorf("IsSpace(U+%04X) = true; want false", c) } } } // Check that the optimizations for IsLetter etc. agree with the tables. // We only need to check the Latin-1 range. func TestLetterOptimizations(t *testing.T) { for i := rune(0); i <= MaxLatin1; i++ { if Is(Letter, i) != IsLetter(i) { t.Errorf("IsLetter(U+%04X) disagrees with Is(Letter)", i) } if Is(Upper, i) != IsUpper(i) { t.Errorf("IsUpper(U+%04X) disagrees with Is(Upper)", i) } if Is(Lower, i) != IsLower(i) { t.Errorf("IsLower(U+%04X) disagrees with Is(Lower)", i) } if Is(Title, i) != IsTitle(i) { t.Errorf("IsTitle(U+%04X) disagrees with Is(Title)", i) } if Is(White_Space, i) != IsSpace(i) { t.Errorf("IsSpace(U+%04X) disagrees with Is(White_Space)", i) } if To(UpperCase, i) != ToUpper(i) { t.Errorf("ToUpper(U+%04X) disagrees with To(Upper)", i) } if To(LowerCase, i) != ToLower(i) { t.Errorf("ToLower(U+%04X) disagrees with To(Lower)", i) } if To(TitleCase, i) != ToTitle(i) { t.Errorf("ToTitle(U+%04X) disagrees with To(Title)", i) } } } func TestTurkishCase(t *testing.T) { lower := []rune("abcçdefgğhıijklmnoöprsştuüvyz") upper := []rune("ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZ") for i, l := range lower { u := upper[i] if TurkishCase.ToLower(l) != l { t.Errorf("lower(U+%04X) is U+%04X not U+%04X", l, TurkishCase.ToLower(l), l) } if TurkishCase.ToUpper(u) != u { t.Errorf("upper(U+%04X) is U+%04X not U+%04X", u, TurkishCase.ToUpper(u), u) } if TurkishCase.ToUpper(l) != u { t.Errorf("upper(U+%04X) is U+%04X not U+%04X", l, TurkishCase.ToUpper(l), u) } if TurkishCase.ToLower(u) != l { t.Errorf("lower(U+%04X) is U+%04X not U+%04X", u, TurkishCase.ToLower(l), l) } if TurkishCase.ToTitle(u) != u { t.Errorf("title(U+%04X) is U+%04X not U+%04X", u, TurkishCase.ToTitle(u), u) } if TurkishCase.ToTitle(l) != u { t.Errorf("title(U+%04X) is U+%04X not U+%04X", l, TurkishCase.ToTitle(l), u) } } } var simpleFoldTests = []string{ // SimpleFold(x) returns the next equivalent rune > x or wraps // around to smaller values. // Easy cases. "Aa", "δΔ", // ASCII special cases. "KkK", "Ssſ", // Non-ASCII special cases. "ρϱΡ", "ͅΙιι", // Extra special cases: has lower/upper but no case fold. "İ", "ı", // Upper comes before lower (Cherokee). "\u13b0\uab80", } func TestSimpleFold(t *testing.T) { for _, tt := range simpleFoldTests { cycle := []rune(tt) r := cycle[len(cycle)-1] for _, out := range cycle { if r := SimpleFold(r); r != out { t.Errorf("SimpleFold(%#U) = %#U, want %#U", r, r, out) } r = out } } } // Running 'go test -calibrate' runs the calibration to find a plausible // cutoff point for linear search of a range list vs. binary search. // We create a fake table and then time how long it takes to do a // sequence of searches within that table, for all possible inputs // relative to the ranges (something before all, in each, between each, after all). // This assumes that all possible runes are equally likely. // In practice most runes are ASCII so this is a conservative estimate // of an effective cutoff value. In practice we could probably set it higher // than what this function recommends. var calibrate = flag.Bool("calibrate", false, "compute crossover for linear vs. binary search") func TestCalibrate(t *testing.T) { if !*calibrate { return } if runtime.GOARCH == "amd64" { fmt.Printf("warning: running calibration on %s\n", runtime.GOARCH) } // Find the point where binary search wins by more than 10%. // The 10% bias gives linear search an edge when they're close, // because on predominantly ASCII inputs linear search is even // better than our benchmarks measure. n := sort.Search(64, func(n int) bool { tab := fakeTable(n) blinear := func(b *testing.B) { tab := tab max := n*5 + 20 for i := 0; i < b.N; i++ { for j := 0; j <= max; j++ { linear(tab, uint16(j)) } } } bbinary := func(b *testing.B) { tab := tab max := n*5 + 20 for i := 0; i < b.N; i++ { for j := 0; j <= max; j++ { binary(tab, uint16(j)) } } } bmlinear := testing.Benchmark(blinear) bmbinary := testing.Benchmark(bbinary) fmt.Printf("n=%d: linear=%d binary=%d\n", n, bmlinear.NsPerOp(), bmbinary.NsPerOp()) return bmlinear.NsPerOp()*100 > bmbinary.NsPerOp()*110 }) fmt.Printf("calibration: linear cutoff = %d\n", n) } func fakeTable(n int) []Range16 { var r16 []Range16 for i := 0; i < n; i++ { r16 = append(r16, Range16{uint16(i*5 + 10), uint16(i*5 + 12), 1}) } return r16 } func linear(ranges []Range16, r uint16) bool { for i := range ranges { range_ := &ranges[i] if r < range_.Lo { return false } if r <= range_.Hi { return (r-range_.Lo)%range_.Stride == 0 } } return false } func binary(ranges []Range16, r uint16) bool { // binary search over ranges lo := 0 hi := len(ranges) for lo < hi { m := lo + (hi-lo)/2 range_ := &ranges[m] if range_.Lo <= r && r <= range_.Hi { return (r-range_.Lo)%range_.Stride == 0 } if r < range_.Lo { hi = m } else { lo = m + 1 } } return false } func TestLatinOffset(t *testing.T) { var maps = []map[string]*RangeTable{ Categories, FoldCategory, FoldScript, Properties, Scripts, } for _, m := range maps { for name, tab := range m { i := 0 for i < len(tab.R16) && tab.R16[i].Hi <= MaxLatin1 { i++ } if tab.LatinOffset != i { t.Errorf("%s: LatinOffset=%d, want %d", name, tab.LatinOffset, i) } } } } ================================================ FILE: examples/go/no_newline_at_eof.go ================================================ // run // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package main func main() { x := 0 func() { x = 1 }() func() { if x != 1 { panic("x != 1") } }() } ================================================ FILE: examples/go/proc.go ================================================ // Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package runtime import ( "runtime/internal/atomic" "runtime/internal/sys" "unsafe" ) var buildVersion = sys.TheVersion // Goroutine scheduler // The scheduler's job is to distribute ready-to-run goroutines over worker threads. // // The main concepts are: // G - goroutine. // M - worker thread, or machine. // P - processor, a resource that is required to execute Go code. // M must have an associated P to execute Go code, however it can be // blocked or in a syscall w/o an associated P. // // Design doc at https://golang.org/s/go11sched. // Worker thread parking/unparking. // We need to balance between keeping enough running worker threads to utilize // available hardware parallelism and parking excessive running worker threads // to conserve CPU resources and power. This is not simple for two reasons: // (1) scheduler state is intentionally distributed (in particular, per-P work // queues), so it is not possible to compute global predicates on fast paths; // (2) for optimal thread management we would need to know the future (don't park // a worker thread when a new goroutine will be readied in near future). // // Three rejected approaches that would work badly: // 1. Centralize all scheduler state (would inhibit scalability). // 2. Direct goroutine handoff. That is, when we ready a new goroutine and there // is a spare P, unpark a thread and handoff it the thread and the goroutine. // This would lead to thread state thrashing, as the thread that readied the // goroutine can be out of work the very next moment, we will need to park it. // Also, it would destroy locality of computation as we want to preserve // dependent goroutines on the same thread; and introduce additional latency. // 3. Unpark an additional thread whenever we ready a goroutine and there is an // idle P, but don't do handoff. This would lead to excessive thread parking/ // unparking as the additional threads will instantly park without discovering // any work to do. // // The current approach: // We unpark an additional thread when we ready a goroutine if (1) there is an // idle P and there are no "spinning" worker threads. A worker thread is considered // spinning if it is out of local work and did not find work in global run queue/ // netpoller; the spinning state is denoted in m.spinning and in sched.nmspinning. // Threads unparked this way are also considered spinning; we don't do goroutine // handoff so such threads are out of work initially. Spinning threads do some // spinning looking for work in per-P run queues before parking. If a spinning // thread finds work it takes itself out of the spinning state and proceeds to // execution. If it does not find work it takes itself out of the spinning state // and then parks. // If there is at least one spinning thread (sched.nmspinning>1), we don't unpark // new threads when readying goroutines. To compensate for that, if the last spinning // thread finds work and stops spinning, it must unpark a new spinning thread. // This approach smooths out unjustified spikes of thread unparking, // but at the same time guarantees eventual maximal CPU parallelism utilization. // // The main implementation complication is that we need to be very careful during // spinning->non-spinning thread transition. This transition can race with submission // of a new goroutine, and either one part or another needs to unpark another worker // thread. If they both fail to do that, we can end up with semi-persistent CPU // underutilization. The general pattern for goroutine readying is: submit a goroutine // to local work queue, #StoreLoad-style memory barrier, check sched.nmspinning. // The general pattern for spinning->non-spinning transition is: decrement nmspinning, // #StoreLoad-style memory barrier, check all per-P work queues for new work. // Note that all this complexity does not apply to global run queue as we are not // sloppy about thread unparking when submitting to global queue. Also see comments // for nmspinning manipulation. var ( m0 m g0 g ) //go:linkname runtime_init runtime.init func runtime_init() //go:linkname main_init main.init func main_init() // main_init_done is a signal used by cgocallbackg that initialization // has been completed. It is made before _cgo_notify_runtime_init_done, // so all cgo calls can rely on it existing. When main_init is complete, // it is closed, meaning cgocallbackg can reliably receive from it. var main_init_done chan bool //go:linkname main_main main.main func main_main() // runtimeInitTime is the nanotime() at which the runtime started. var runtimeInitTime int64 // Value to use for signal mask for newly created M's. var initSigmask sigset // The main goroutine. func main() { g := getg() // Racectx of m0->g0 is used only as the parent of the main goroutine. // It must not be used for anything else. g.m.g0.racectx = 0 // Max stack size is 1 GB on 64-bit, 250 MB on 32-bit. // Using decimal instead of binary GB and MB because // they look nicer in the stack overflow failure message. if sys.PtrSize == 8 { maxstacksize = 1000000000 } else { maxstacksize = 250000000 } // Record when the world started. runtimeInitTime = nanotime() systemstack(func() { newm(sysmon, nil) }) // Lock the main goroutine onto this, the main OS thread, // during initialization. Most programs won't care, but a few // do require certain calls to be made by the main thread. // Those can arrange for main.main to run in the main thread // by calling runtime.LockOSThread during initialization // to preserve the lock. lockOSThread() if g.m != &m0 { throw("runtime.main not on m0") } runtime_init() // must be before defer // Defer unlock so that runtime.Goexit during init does the unlock too. needUnlock := true defer func() { if needUnlock { unlockOSThread() } }() gcenable() main_init_done = make(chan bool) if iscgo { if _cgo_thread_start == nil { throw("_cgo_thread_start missing") } if _cgo_malloc == nil { throw("_cgo_malloc missing") } if _cgo_free == nil { throw("_cgo_free missing") } if GOOS != "windows" { if _cgo_setenv == nil { throw("_cgo_setenv missing") } if _cgo_unsetenv == nil { throw("_cgo_unsetenv missing") } } if _cgo_notify_runtime_init_done == nil { throw("_cgo_notify_runtime_init_done missing") } cgocall(_cgo_notify_runtime_init_done, nil) } main_init() close(main_init_done) needUnlock = false unlockOSThread() if isarchive || islibrary { // A program compiled with -buildmode=c-archive or c-shared // has a main, but it is not executed. return } main_main() if raceenabled { racefini() } // Make racy client program work: if panicking on // another goroutine at the same time as main returns, // let the other goroutine finish printing the panic trace. // Once it does, it will exit. See issue 3934. if panicking != 0 { gopark(nil, nil, "panicwait", traceEvGoStop, 1) } exit(0) for { var x *int32 *x = 0 } } // os_beforeExit is called from os.Exit(0). //go:linkname os_beforeExit os.runtime_beforeExit func os_beforeExit() { if raceenabled { racefini() } } // start forcegc helper goroutine func init() { go forcegchelper() } func forcegchelper() { forcegc.g = getg() for { lock(&forcegc.lock) if forcegc.idle != 0 { throw("forcegc: phase error") } atomic.Store(&forcegc.idle, 1) goparkunlock(&forcegc.lock, "force gc (idle)", traceEvGoBlock, 1) // this goroutine is explicitly resumed by sysmon if debug.gctrace > 0 { println("GC forced") } gcStart(gcBackgroundMode, true) } } //go:nosplit // Gosched yields the processor, allowing other goroutines to run. It does not // suspend the current goroutine, so execution resumes automatically. func Gosched() { mcall(gosched_m) } // Puts the current goroutine into a waiting state and calls unlockf. // If unlockf returns false, the goroutine is resumed. func gopark(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason string, traceEv byte, traceskip int) { mp := acquirem() gp := mp.curg status := readgstatus(gp) if status != _Grunning && status != _Gscanrunning { throw("gopark: bad g status") } mp.waitlock = lock mp.waitunlockf = *(*unsafe.Pointer)(unsafe.Pointer(&unlockf)) gp.waitreason = reason mp.waittraceev = traceEv mp.waittraceskip = traceskip releasem(mp) // can't do anything that might move the G between Ms here. mcall(park_m) } // Puts the current goroutine into a waiting state and unlocks the lock. // The goroutine can be made runnable again by calling goready(gp). func goparkunlock(lock *mutex, reason string, traceEv byte, traceskip int) { gopark(parkunlock_c, unsafe.Pointer(lock), reason, traceEv, traceskip) } func goready(gp *g, traceskip int) { systemstack(func() { ready(gp, traceskip) }) } //go:nosplit func acquireSudog() *sudog { // Delicate dance: the semaphore implementation calls // acquireSudog, acquireSudog calls new(sudog), // new calls malloc, malloc can call the garbage collector, // and the garbage collector calls the semaphore implementation // in stopTheWorld. // Break the cycle by doing acquirem/releasem around new(sudog). // The acquirem/releasem increments m.locks during new(sudog), // which keeps the garbage collector from being invoked. mp := acquirem() pp := mp.p.ptr() if len(pp.sudogcache) == 0 { lock(&sched.sudoglock) // First, try to grab a batch from central cache. for len(pp.sudogcache) < cap(pp.sudogcache)/2 && sched.sudogcache != nil { s := sched.sudogcache sched.sudogcache = s.next s.next = nil pp.sudogcache = append(pp.sudogcache, s) } unlock(&sched.sudoglock) // If the central cache is empty, allocate a new one. if len(pp.sudogcache) == 0 { pp.sudogcache = append(pp.sudogcache, new(sudog)) } } n := len(pp.sudogcache) s := pp.sudogcache[n-1] pp.sudogcache[n-1] = nil pp.sudogcache = pp.sudogcache[:n-1] if s.elem != nil { throw("acquireSudog: found s.elem != nil in cache") } releasem(mp) return s } //go:nosplit func releaseSudog(s *sudog) { if s.elem != nil { throw("runtime: sudog with non-nil elem") } if s.selectdone != nil { throw("runtime: sudog with non-nil selectdone") } if s.next != nil { throw("runtime: sudog with non-nil next") } if s.prev != nil { throw("runtime: sudog with non-nil prev") } if s.waitlink != nil { throw("runtime: sudog with non-nil waitlink") } gp := getg() if gp.param != nil { throw("runtime: releaseSudog with non-nil gp.param") } mp := acquirem() // avoid rescheduling to another P pp := mp.p.ptr() if len(pp.sudogcache) == cap(pp.sudogcache) { // Transfer half of local cache to the central cache. var first, last *sudog for len(pp.sudogcache) > cap(pp.sudogcache)/2 { n := len(pp.sudogcache) p := pp.sudogcache[n-1] pp.sudogcache[n-1] = nil pp.sudogcache = pp.sudogcache[:n-1] if first == nil { first = p } else { last.next = p } last = p } lock(&sched.sudoglock) last.next = sched.sudogcache sched.sudogcache = first unlock(&sched.sudoglock) } pp.sudogcache = append(pp.sudogcache, s) releasem(mp) } // funcPC returns the entry PC of the function f. // It assumes that f is a func value. Otherwise the behavior is undefined. //go:nosplit func funcPC(f interface{}) uintptr { return **(**uintptr)(add(unsafe.Pointer(&f), sys.PtrSize)) } // called from assembly func badmcall(fn func(*g)) { throw("runtime: mcall called on m->g0 stack") } func badmcall2(fn func(*g)) { throw("runtime: mcall function returned") } func badreflectcall() { panic("runtime: arg size to reflect.call more than 1GB") } func lockedOSThread() bool { gp := getg() return gp.lockedm != nil && gp.m.lockedg != nil } var ( allgs []*g allglock mutex ) func allgadd(gp *g) { if readgstatus(gp) == _Gidle { throw("allgadd: bad status Gidle") } lock(&allglock) allgs = append(allgs, gp) allglen = uintptr(len(allgs)) unlock(&allglock) } const ( // Number of goroutine ids to grab from sched.goidgen to local per-P cache at once. // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number. _GoidCacheBatch = 16 ) // The bootstrap sequence is: // // call osinit // call schedinit // make & queue new G // call runtime·mstart // // The new G calls runtime·main. func schedinit() { // raceinit must be the first call to race detector. // In particular, it must be done before mallocinit below calls racemapshadow. _g_ := getg() if raceenabled { _g_.racectx = raceinit() } sched.maxmcount = 10000 // Cache the framepointer experiment. This affects stack unwinding. framepointer_enabled = haveexperiment("framepointer") tracebackinit() moduledataverify() stackinit() mallocinit() mcommoninit(_g_.m) msigsave(_g_.m) initSigmask = _g_.m.sigmask goargs() goenvs() parsedebugvars() gcinit() sched.lastpoll = uint64(nanotime()) procs := int(ncpu) if n := atoi(gogetenv("GOMAXPROCS")); n > 0 { if n > _MaxGomaxprocs { n = _MaxGomaxprocs } procs = n } if procresize(int32(procs)) != nil { throw("unknown runnable goroutine during bootstrap") } if buildVersion == "" { // Condition should never trigger. This code just serves // to ensure runtime·buildVersion is kept in the resulting binary. buildVersion = "unknown" } } func dumpgstatus(gp *g) { _g_ := getg() print("runtime: gp: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n") print("runtime: g: g=", _g_, ", goid=", _g_.goid, ", g->atomicstatus=", readgstatus(_g_), "\n") } func checkmcount() { // sched lock is held if sched.mcount > sched.maxmcount { print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n") throw("thread exhaustion") } } func mcommoninit(mp *m) { _g_ := getg() // g0 stack won't make sense for user (and is not necessary unwindable). if _g_ != _g_.m.g0 { callers(1, mp.createstack[:]) } mp.fastrand = 0x49f6428a + uint32(mp.id) + uint32(cputicks()) if mp.fastrand == 0 { mp.fastrand = 0x49f6428a } lock(&sched.lock) mp.id = sched.mcount sched.mcount++ checkmcount() mpreinit(mp) if mp.gsignal != nil { mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard } // Add to allm so garbage collector doesn't free g->m // when it is just in a register or thread-local storage. mp.alllink = allm // NumCgoCall() iterates over allm w/o schedlock, // so we need to publish it safely. atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp)) unlock(&sched.lock) } // Mark gp ready to run. func ready(gp *g, traceskip int) { if trace.enabled { traceGoUnpark(gp, traceskip) } status := readgstatus(gp) // Mark runnable. _g_ := getg() _g_.m.locks++ // disable preemption because it can be holding p in a local var if status&^_Gscan != _Gwaiting { dumpgstatus(gp) throw("bad g->status in ready") } // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq casgstatus(gp, _Gwaiting, _Grunnable) runqput(_g_.m.p.ptr(), gp, true) if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 { // TODO: fast atomic wakep() } _g_.m.locks-- if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in Case we've cleared it in newstack _g_.stackguard0 = stackPreempt } } func gcprocs() int32 { // Figure out how many CPUs to use during GC. // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. lock(&sched.lock) n := gomaxprocs if n > ncpu { n = ncpu } if n > _MaxGcproc { n = _MaxGcproc } if n > sched.nmidle+1 { // one M is currently running n = sched.nmidle + 1 } unlock(&sched.lock) return n } func needaddgcproc() bool { lock(&sched.lock) n := gomaxprocs if n > ncpu { n = ncpu } if n > _MaxGcproc { n = _MaxGcproc } n -= sched.nmidle + 1 // one M is currently running unlock(&sched.lock) return n > 0 } func helpgc(nproc int32) { _g_ := getg() lock(&sched.lock) pos := 0 for n := int32(1); n < nproc; n++ { // one M is currently running if allp[pos].mcache == _g_.m.mcache { pos++ } mp := mget() if mp == nil { throw("gcprocs inconsistency") } mp.helpgc = n mp.p.set(allp[pos]) mp.mcache = allp[pos].mcache pos++ notewakeup(&mp.park) } unlock(&sched.lock) } // freezeStopWait is a large value that freezetheworld sets // sched.stopwait to in order to request that all Gs permanently stop. const freezeStopWait = 0x7fffffff // Similar to stopTheWorld but best-effort and can be called several times. // There is no reverse operation, used during crashing. // This function must not lock any mutexes. func freezetheworld() { // stopwait and preemption requests can be lost // due to races with concurrently executing threads, // so try several times for i := 0; i < 5; i++ { // this should tell the scheduler to not start any new goroutines sched.stopwait = freezeStopWait atomic.Store(&sched.gcwaiting, 1) // this should stop running goroutines if !preemptall() { break // no running goroutines } usleep(1000) } // to be sure usleep(1000) preemptall() usleep(1000) } func isscanstatus(status uint32) bool { if status == _Gscan { throw("isscanstatus: Bad status Gscan") } return status&_Gscan == _Gscan } // All reads and writes of g's status go through readgstatus, casgstatus // castogscanstatus, casfrom_Gscanstatus. //go:nosplit func readgstatus(gp *g) uint32 { return atomic.Load(&gp.atomicstatus) } // Ownership of gscanvalid: // // If gp is running (meaning status == _Grunning or _Grunning|_Gscan), // then gp owns gp.gscanvalid, and other goroutines must not modify it. // // Otherwise, a second goroutine can lock the scan state by setting _Gscan // in the status bit and then modify gscanvalid, and then unlock the scan state. // // Note that the first condition implies an exception to the second: // if a second goroutine changes gp's status to _Grunning|_Gscan, // that second goroutine still does not have the right to modify gscanvalid. // The Gscanstatuses are acting like locks and this releases them. // If it proves to be a performance hit we should be able to make these // simple atomic stores but for now we are going to throw if // we see an inconsistent state. func casfrom_Gscanstatus(gp *g, oldval, newval uint32) { success := false // Check that transition is valid. switch oldval { default: print("runtime: casfrom_Gscanstatus bad oldval gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n") dumpgstatus(gp) throw("casfrom_Gscanstatus:top gp->status is not in scan state") case _Gscanrunnable, _Gscanwaiting, _Gscanrunning, _Gscansyscall: if newval == oldval&^_Gscan { success = atomic.Cas(&gp.atomicstatus, oldval, newval) } case _Gscanenqueue: if newval == _Gwaiting { success = atomic.Cas(&gp.atomicstatus, oldval, newval) } } if !success { print("runtime: casfrom_Gscanstatus failed gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n") dumpgstatus(gp) throw("casfrom_Gscanstatus: gp->status is not in scan state") } if newval == _Grunning { gp.gcscanvalid = false } } // This will return false if the gp is not in the expected status and the cas fails. // This acts like a lock acquire while the casfromgstatus acts like a lock release. func castogscanstatus(gp *g, oldval, newval uint32) bool { switch oldval { case _Grunnable, _Gwaiting, _Gsyscall: if newval == oldval|_Gscan { return atomic.Cas(&gp.atomicstatus, oldval, newval) } case _Grunning: if newval == _Gscanrunning || newval == _Gscanenqueue { return atomic.Cas(&gp.atomicstatus, oldval, newval) } } print("runtime: castogscanstatus oldval=", hex(oldval), " newval=", hex(newval), "\n") throw("castogscanstatus") panic("not reached") } // If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus // and casfrom_Gscanstatus instead. // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that // put it in the Gscan state is finished. //go:nosplit func casgstatus(gp *g, oldval, newval uint32) { if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval { systemstack(func() { print("runtime: casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n") throw("casgstatus: bad incoming values") }) } if oldval == _Grunning && gp.gcscanvalid { // If oldvall == _Grunning, then the actual status must be // _Grunning or _Grunning|_Gscan; either way, // we own gp.gcscanvalid, so it's safe to read. // gp.gcscanvalid must not be true when we are running. print("runtime: casgstatus ", hex(oldval), "->", hex(newval), " gp.status=", hex(gp.atomicstatus), " gp.gcscanvalid=true\n") throw("casgstatus") } // loop if gp->atomicstatus is in a scan state giving // GC time to finish and change the state to oldval. for !atomic.Cas(&gp.atomicstatus, oldval, newval) { if oldval == _Gwaiting && gp.atomicstatus == _Grunnable { systemstack(func() { throw("casgstatus: waiting for Gwaiting but is Grunnable") }) } // Help GC if needed. // if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) { // gp.preemptscan = false // systemstack(func() { // gcphasework(gp) // }) // } } if newval == _Grunning { gp.gcscanvalid = false } } // casgstatus(gp, oldstatus, Gcopystack), assuming oldstatus is Gwaiting or Grunnable. // Returns old status. Cannot call casgstatus directly, because we are racing with an // async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus, // it might have become Grunnable by the time we get to the cas. If we called casgstatus, // it would loop waiting for the status to go back to Gwaiting, which it never will. //go:nosplit func casgcopystack(gp *g) uint32 { for { oldstatus := readgstatus(gp) &^ _Gscan if oldstatus != _Gwaiting && oldstatus != _Grunnable { throw("copystack: bad status, not Gwaiting or Grunnable") } if atomic.Cas(&gp.atomicstatus, oldstatus, _Gcopystack) { return oldstatus } } } // scang blocks until gp's stack has been scanned. // It might be scanned by scang or it might be scanned by the goroutine itself. // Either way, the stack scan has completed when scang returns. func scang(gp *g) { // Invariant; we (the caller, markroot for a specific goroutine) own gp.gcscandone. // Nothing is racing with us now, but gcscandone might be set to true left over // from an earlier round of stack scanning (we scan twice per GC). // We use gcscandone to record whether the scan has been done during this round. // It is important that the scan happens exactly once: if called twice, // the installation of stack barriers will detect the double scan and die. gp.gcscandone = false // Endeavor to get gcscandone set to true, // either by doing the stack scan ourselves or by coercing gp to scan itself. // gp.gcscandone can transition from false to true when we're not looking // (if we asked for preemption), so any time we lock the status using // castogscanstatus we have to double-check that the scan is still not done. for !gp.gcscandone { switch s := readgstatus(gp); s { default: dumpgstatus(gp) throw("stopg: invalid status") case _Gdead: // No stack. gp.gcscandone = true case _Gcopystack: // Stack being switched. Go around again. case _Grunnable, _Gsyscall, _Gwaiting: // Claim goroutine by setting scan bit. // Racing with execution or readying of gp. // The scan bit keeps them from running // the goroutine until we're done. if castogscanstatus(gp, s, s|_Gscan) { if !gp.gcscandone { scanstack(gp) gp.gcscandone = true } restartg(gp) } case _Gscanwaiting: // newstack is doing a scan for us right now. Wait. case _Grunning: // Goroutine running. Try to preempt execution so it can scan itself. // The preemption handler (in newstack) does the actual scan. // Optimization: if there is already a pending preemption request // (from the previous loop iteration), don't bother with the atomics. if gp.preemptscan && gp.preempt && gp.stackguard0 == stackPreempt { break } // Ask for preemption and self scan. if castogscanstatus(gp, _Grunning, _Gscanrunning) { if !gp.gcscandone { gp.preemptscan = true gp.preempt = true gp.stackguard0 = stackPreempt } casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning) } } } gp.preemptscan = false // cancel scan request if no longer needed } // The GC requests that this routine be moved from a scanmumble state to a mumble state. func restartg(gp *g) { s := readgstatus(gp) switch s { default: dumpgstatus(gp) throw("restartg: unexpected status") case _Gdead: // ok case _Gscanrunnable, _Gscanwaiting, _Gscansyscall: casfrom_Gscanstatus(gp, s, s&^_Gscan) // Scan is now completed. // Goroutine now needs to be made runnable. // We put it on the global run queue; ready blocks on the global scheduler lock. case _Gscanenqueue: casfrom_Gscanstatus(gp, _Gscanenqueue, _Gwaiting) if gp != getg().m.curg { throw("processing Gscanenqueue on wrong m") } dropg() ready(gp, 0) } } // stopTheWorld stops all P's from executing goroutines, interrupting // all goroutines at GC safe points and records reason as the reason // for the stop. On return, only the current goroutine's P is running. // stopTheWorld must not be called from a system stack and the caller // must not hold worldsema. The caller must call startTheWorld when // other P's should resume execution. // // stopTheWorld is safe for multiple goroutines to call at the // same time. Each will execute its own stop, and the stops will // be serialized. // // This is also used by routines that do stack dumps. If the system is // in panic or being exited, this may not reliably stop all // goroutines. func stopTheWorld(reason string) { semacquire(&worldsema, false) getg().m.preemptoff = reason systemstack(stopTheWorldWithSema) } // startTheWorld undoes the effects of stopTheWorld. func startTheWorld() { systemstack(startTheWorldWithSema) // worldsema must be held over startTheWorldWithSema to ensure // gomaxprocs cannot change while worldsema is held. semrelease(&worldsema) getg().m.preemptoff = "" } // Holding worldsema grants an M the right to try to stop the world // and prevents gomaxprocs from changing concurrently. var worldsema uint32 = 1 // stopTheWorldWithSema is the core implementation of stopTheWorld. // The caller is responsible for acquiring worldsema and disabling // preemption first and then should stopTheWorldWithSema on the system // stack: // // semacquire(&worldsema, false) // m.preemptoff = "reason" // systemstack(stopTheWorldWithSema) // // When finished, the caller must either call startTheWorld or undo // these three operations separately: // // m.preemptoff = "" // systemstack(startTheWorldWithSema) // semrelease(&worldsema) // // It is allowed to acquire worldsema once and then execute multiple // startTheWorldWithSema/stopTheWorldWithSema pairs. // Other P's are able to execute between successive calls to // startTheWorldWithSema and stopTheWorldWithSema. // Holding worldsema causes any other goroutines invoking // stopTheWorld to block. func stopTheWorldWithSema() { _g_ := getg() // If we hold a lock, then we won't be able to stop another M // that is blocked trying to acquire the lock. if _g_.m.locks > 0 { throw("stopTheWorld: holding locks") } lock(&sched.lock) sched.stopwait = gomaxprocs atomic.Store(&sched.gcwaiting, 1) preemptall() // stop current P _g_.m.p.ptr().status = _Pgcstop // Pgcstop is only diagnostic. sched.stopwait-- // try to retake all P's in Psyscall status for i := 0; i < int(gomaxprocs); i++ { p := allp[i] s := p.status if s == _Psyscall && atomic.Cas(&p.status, s, _Pgcstop) { if trace.enabled { traceGoSysBlock(p) traceProcStop(p) } p.syscalltick++ sched.stopwait-- } } // stop idle P's for { p := pidleget() if p == nil { break } p.status = _Pgcstop sched.stopwait-- } wait := sched.stopwait > 0 unlock(&sched.lock) // wait for remaining P's to stop voluntarily if wait { for { // wait for 100us, then try to re-preempt in case of any races if notetsleep(&sched.stopnote, 100*1000) { noteclear(&sched.stopnote) break } preemptall() } } if sched.stopwait != 0 { throw("stopTheWorld: not stopped") } for i := 0; i < int(gomaxprocs); i++ { p := allp[i] if p.status != _Pgcstop { throw("stopTheWorld: not stopped") } } } func mhelpgc() { _g_ := getg() _g_.m.helpgc = -1 } func startTheWorldWithSema() { _g_ := getg() _g_.m.locks++ // disable preemption because it can be holding p in a local var gp := netpoll(false) // non-blocking injectglist(gp) add := needaddgcproc() lock(&sched.lock) procs := gomaxprocs if newprocs != 0 { procs = newprocs newprocs = 0 } p1 := procresize(procs) sched.gcwaiting = 0 if sched.sysmonwait != 0 { sched.sysmonwait = 0 notewakeup(&sched.sysmonnote) } unlock(&sched.lock) for p1 != nil { p := p1 p1 = p1.link.ptr() if p.m != 0 { mp := p.m.ptr() p.m = 0 if mp.nextp != 0 { throw("startTheWorld: inconsistent mp->nextp") } mp.nextp.set(p) notewakeup(&mp.park) } else { // Start M to run P. Do not start another M below. newm(nil, p) add = false } } // Wakeup an additional proc in case we have excessive runnable goroutines // in local queues or in the global queue. If we don't, the proc will park itself. // If we have lots of excessive work, resetspinning will unpark additional procs as necessary. if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 { wakep() } if add { // If GC could have used another helper proc, start one now, // in the hope that it will be available next time. // It would have been even better to start it before the collection, // but doing so requires allocating memory, so it's tricky to // coordinate. This lazy approach works out in practice: // we don't mind if the first couple gc rounds don't have quite // the maximum number of procs. newm(mhelpgc, nil) } _g_.m.locks-- if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack _g_.stackguard0 = stackPreempt } } // Called to start an M. //go:nosplit func mstart() { _g_ := getg() if _g_.stack.lo == 0 { // Initialize stack bounds from system stack. // Cgo may have left stack size in stack.hi. size := _g_.stack.hi if size == 0 { size = 8192 * sys.StackGuardMultiplier } _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&size))) _g_.stack.lo = _g_.stack.hi - size + 1024 } // Initialize stack guards so that we can start calling // both Go and C functions with stack growth prologues. _g_.stackguard0 = _g_.stack.lo + _StackGuard _g_.stackguard1 = _g_.stackguard0 mstart1() } func mstart1() { _g_ := getg() if _g_ != _g_.m.g0 { throw("bad runtime·mstart") } // Record top of stack for use by mcall. // Once we call schedule we're never coming back, // so other calls can reuse this stack space. gosave(&_g_.m.g0.sched) _g_.m.g0.sched.pc = ^uintptr(0) // make sure it is never used asminit() minit() // Install signal handlers; after minit so that minit can // prepare the thread to be able to handle the signals. if _g_.m == &m0 { // Create an extra M for callbacks on threads not created by Go. if iscgo && !cgoHasExtraM { cgoHasExtraM = true newextram() } initsig(false) } if fn := _g_.m.mstartfn; fn != nil { fn() } if _g_.m.helpgc != 0 { _g_.m.helpgc = 0 stopm() } else if _g_.m != &m0 { acquirep(_g_.m.nextp.ptr()) _g_.m.nextp = 0 } schedule() } // forEachP calls fn(p) for every P p when p reaches a GC safe point. // If a P is currently executing code, this will bring the P to a GC // safe point and execute fn on that P. If the P is not executing code // (it is idle or in a syscall), this will call fn(p) directly while // preventing the P from exiting its state. This does not ensure that // fn will run on every CPU executing Go code, but it acts as a global // memory barrier. GC uses this as a "ragged barrier." // // The caller must hold worldsema. // //go:systemstack func forEachP(fn func(*p)) { mp := acquirem() _p_ := getg().m.p.ptr() lock(&sched.lock) if sched.safePointWait != 0 { throw("forEachP: sched.safePointWait != 0") } sched.safePointWait = gomaxprocs - 1 sched.safePointFn = fn // Ask all Ps to run the safe point function. for _, p := range allp[:gomaxprocs] { if p != _p_ { atomic.Store(&p.runSafePointFn, 1) } } preemptall() // Any P entering _Pidle or _Psyscall from now on will observe // p.runSafePointFn == 1 and will call runSafePointFn when // changing its status to _Pidle/_Psyscall. // Run safe point function for all idle Ps. sched.pidle will // not change because we hold sched.lock. for p := sched.pidle.ptr(); p != nil; p = p.link.ptr() { if atomic.Cas(&p.runSafePointFn, 1, 0) { fn(p) sched.safePointWait-- } } wait := sched.safePointWait > 0 unlock(&sched.lock) // Run fn for the current P. fn(_p_) // Force Ps currently in _Psyscall into _Pidle and hand them // off to induce safe point function execution. for i := 0; i < int(gomaxprocs); i++ { p := allp[i] s := p.status if s == _Psyscall && p.runSafePointFn == 1 && atomic.Cas(&p.status, s, _Pidle) { if trace.enabled { traceGoSysBlock(p) traceProcStop(p) } p.syscalltick++ handoffp(p) } } // Wait for remaining Ps to run fn. if wait { for { // Wait for 100us, then try to re-preempt in // case of any races. // // Requires system stack. if notetsleep(&sched.safePointNote, 100*1000) { noteclear(&sched.safePointNote) break } preemptall() } } if sched.safePointWait != 0 { throw("forEachP: not done") } for i := 0; i < int(gomaxprocs); i++ { p := allp[i] if p.runSafePointFn != 0 { throw("forEachP: P did not run fn") } } lock(&sched.lock) sched.safePointFn = nil unlock(&sched.lock) releasem(mp) } // runSafePointFn runs the safe point function, if any, for this P. // This should be called like // // if getg().m.p.runSafePointFn != 0 { // runSafePointFn() // } // // runSafePointFn must be checked on any transition in to _Pidle or // _Psyscall to avoid a race where forEachP sees that the P is running // just before the P goes into _Pidle/_Psyscall and neither forEachP // nor the P run the safe-point function. func runSafePointFn() { p := getg().m.p.ptr() // Resolve the race between forEachP running the safe-point // function on this P's behalf and this P running the // safe-point function directly. if !atomic.Cas(&p.runSafePointFn, 1, 0) { return } sched.safePointFn(p) lock(&sched.lock) sched.safePointWait-- if sched.safePointWait == 0 { notewakeup(&sched.safePointNote) } unlock(&sched.lock) } // When running with cgo, we call _cgo_thread_start // to start threads for us so that we can play nicely with // foreign code. var cgoThreadStart unsafe.Pointer type cgothreadstart struct { g guintptr tls *uint64 fn unsafe.Pointer } // Allocate a new m unassociated with any thread. // Can use p for allocation context if needed. // fn is recorded as the new m's m.mstartfn. // // This function it known to the compiler to inhibit the // go:nowritebarrierrec annotation because it uses P for allocation. func allocm(_p_ *p, fn func()) *m { _g_ := getg() _g_.m.locks++ // disable GC because it can be called from sysmon if _g_.m.p == 0 { acquirep(_p_) // temporarily borrow p for mallocs in this function } mp := new(m) mp.mstartfn = fn mcommoninit(mp) // In case of cgo or Solaris, pthread_create will make us a stack. // Windows and Plan 9 will layout sched stack on OS stack. if iscgo || GOOS == "solaris" || GOOS == "windows" || GOOS == "plan9" { mp.g0 = malg(-1) } else { mp.g0 = malg(8192 * sys.StackGuardMultiplier) } mp.g0.m = mp if _p_ == _g_.m.p.ptr() { releasep() } _g_.m.locks-- if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack _g_.stackguard0 = stackPreempt } return mp } // needm is called when a cgo callback happens on a // thread without an m (a thread not created by Go). // In this case, needm is expected to find an m to use // and return with m, g initialized correctly. // Since m and g are not set now (likely nil, but see below) // needm is limited in what routines it can call. In particular // it can only call nosplit functions (textflag 7) and cannot // do any scheduling that requires an m. // // In order to avoid needing heavy lifting here, we adopt // the following strategy: there is a stack of available m's // that can be stolen. Using compare-and-swap // to pop from the stack has ABA races, so we simulate // a lock by doing an exchange (via casp) to steal the stack // head and replace the top pointer with MLOCKED (1). // This serves as a simple spin lock that we can use even // without an m. The thread that locks the stack in this way // unlocks the stack by storing a valid stack head pointer. // // In order to make sure that there is always an m structure // available to be stolen, we maintain the invariant that there // is always one more than needed. At the beginning of the // program (if cgo is in use) the list is seeded with a single m. // If needm finds that it has taken the last m off the list, its job // is - once it has installed its own m so that it can do things like // allocate memory - to create a spare m and put it on the list. // // Each of these extra m's also has a g0 and a curg that are // pressed into service as the scheduling stack and current // goroutine for the duration of the cgo callback. // // When the callback is done with the m, it calls dropm to // put the m back on the list. //go:nosplit func needm(x byte) { if iscgo && !cgoHasExtraM { // Can happen if C/C++ code calls Go from a global ctor. // Can not throw, because scheduler is not initialized yet. write(2, unsafe.Pointer(&earlycgocallback[0]), int32(len(earlycgocallback))) exit(1) } // Lock extra list, take head, unlock popped list. // nilokay=false is safe here because of the invariant above, // that the extra list always contains or will soon contain // at least one m. mp := lockextra(false) // Set needextram when we've just emptied the list, // so that the eventual call into cgocallbackg will // allocate a new m for the extra list. We delay the // allocation until then so that it can be done // after exitsyscall makes sure it is okay to be // running at all (that is, there's no garbage collection // running right now). mp.needextram = mp.schedlink == 0 unlockextra(mp.schedlink.ptr()) // Save and block signals before installing g. // Once g is installed, any incoming signals will try to execute, // but we won't have the sigaltstack settings and other data // set up appropriately until the end of minit, which will // unblock the signals. This is the same dance as when // starting a new m to run Go code via newosproc. msigsave(mp) sigblock() // Install g (= m->g0) and set the stack bounds // to match the current stack. We don't actually know // how big the stack is, like we don't know how big any // scheduling stack is, but we assume there's at least 32 kB, // which is more than enough for us. setg(mp.g0) _g_ := getg() _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&x))) + 1024 _g_.stack.lo = uintptr(noescape(unsafe.Pointer(&x))) - 32*1024 _g_.stackguard0 = _g_.stack.lo + _StackGuard // Initialize this thread to use the m. asminit() minit() } var earlycgocallback = []byte("fatal error: cgo callback before cgo call\n") // newextram allocates an m and puts it on the extra list. // It is called with a working local m, so that it can do things // like call schedlock and allocate. func newextram() { // Create extra goroutine locked to extra m. // The goroutine is the context in which the cgo callback will run. // The sched.pc will never be returned to, but setting it to // goexit makes clear to the traceback routines where // the goroutine stack ends. mp := allocm(nil, nil) gp := malg(4096) gp.sched.pc = funcPC(goexit) + sys.PCQuantum gp.sched.sp = gp.stack.hi gp.sched.sp -= 4 * sys.RegSize // extra space in case of reads slightly beyond frame gp.sched.lr = 0 gp.sched.g = guintptr(unsafe.Pointer(gp)) gp.syscallpc = gp.sched.pc gp.syscallsp = gp.sched.sp gp.stktopsp = gp.sched.sp // malg returns status as Gidle, change to Gsyscall before adding to allg // where GC will see it. casgstatus(gp, _Gidle, _Gsyscall) gp.m = mp mp.curg = gp mp.locked = _LockInternal mp.lockedg = gp gp.lockedm = mp gp.goid = int64(atomic.Xadd64(&sched.goidgen, 1)) if raceenabled { gp.racectx = racegostart(funcPC(newextram)) } // put on allg for garbage collector allgadd(gp) // Add m to the extra list. mnext := lockextra(true) mp.schedlink.set(mnext) unlockextra(mp) } // dropm is called when a cgo callback has called needm but is now // done with the callback and returning back into the non-Go thread. // It puts the current m back onto the extra list. // // The main expense here is the call to signalstack to release the // m's signal stack, and then the call to needm on the next callback // from this thread. It is tempting to try to save the m for next time, // which would eliminate both these costs, but there might not be // a next time: the current thread (which Go does not control) might exit. // If we saved the m for that thread, there would be an m leak each time // such a thread exited. Instead, we acquire and release an m on each // call. These should typically not be scheduling operations, just a few // atomics, so the cost should be small. // // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread // variable using pthread_key_create. Unlike the pthread keys we already use // on OS X, this dummy key would never be read by Go code. It would exist // only so that we could register at thread-exit-time destructor. // That destructor would put the m back onto the extra list. // This is purely a performance optimization. The current version, // in which dropm happens on each cgo call, is still correct too. // We may have to keep the current version on systems with cgo // but without pthreads, like Windows. func dropm() { // Clear m and g, and return m to the extra list. // After the call to setg we can only call nosplit functions // with no pointer manipulation. mp := getg().m // Block signals before unminit. // Unminit unregisters the signal handling stack (but needs g on some systems). // Setg(nil) clears g, which is the signal handler's cue not to run Go handlers. // It's important not to try to handle a signal between those two steps. sigmask := mp.sigmask sigblock() unminit() mnext := lockextra(true) mp.schedlink.set(mnext) setg(nil) // Commit the release of mp. unlockextra(mp) msigrestore(sigmask) } // A helper function for EnsureDropM. func getm() uintptr { return uintptr(unsafe.Pointer(getg().m)) } var extram uintptr // lockextra locks the extra list and returns the list head. // The caller must unlock the list by storing a new list head // to extram. If nilokay is true, then lockextra will // return a nil list head if that's what it finds. If nilokay is false, // lockextra will keep waiting until the list head is no longer nil. //go:nosplit func lockextra(nilokay bool) *m { const locked = 1 for { old := atomic.Loaduintptr(&extram) if old == locked { yield := osyield yield() continue } if old == 0 && !nilokay { usleep(1) continue } if atomic.Casuintptr(&extram, old, locked) { return (*m)(unsafe.Pointer(old)) } yield := osyield yield() continue } } //go:nosplit func unlockextra(mp *m) { atomic.Storeuintptr(&extram, uintptr(unsafe.Pointer(mp))) } // Create a new m. It will start off with a call to fn, or else the scheduler. // fn needs to be static and not a heap allocated closure. // May run with m.p==nil, so write barriers are not allowed. //go:nowritebarrier func newm(fn func(), _p_ *p) { mp := allocm(_p_, fn) mp.nextp.set(_p_) mp.sigmask = initSigmask if iscgo { var ts cgothreadstart if _cgo_thread_start == nil { throw("_cgo_thread_start missing") } ts.g.set(mp.g0) ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0])) ts.fn = unsafe.Pointer(funcPC(mstart)) if msanenabled { msanwrite(unsafe.Pointer(&ts), unsafe.Sizeof(ts)) } asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts)) return } newosproc(mp, unsafe.Pointer(mp.g0.stack.hi)) } // Stops execution of the current m until new work is available. // Returns with acquired P. func stopm() { _g_ := getg() if _g_.m.locks != 0 { throw("stopm holding locks") } if _g_.m.p != 0 { throw("stopm holding p") } if _g_.m.spinning { throw("stopm spinning") } retry: lock(&sched.lock) mput(_g_.m) unlock(&sched.lock) notesleep(&_g_.m.park) noteclear(&_g_.m.park) if _g_.m.helpgc != 0 { gchelper() _g_.m.helpgc = 0 _g_.m.mcache = nil _g_.m.p = 0 goto retry } acquirep(_g_.m.nextp.ptr()) _g_.m.nextp = 0 } func mspinning() { // startm's caller incremented nmspinning. Set the new M's spinning. getg().m.spinning = true } // Schedules some M to run the p (creates an M if necessary). // If p==nil, tries to get an idle P, if no idle P's does nothing. // May run with m.p==nil, so write barriers are not allowed. // If spinning is set, the caller has incremented nmspinning and startm will // either decrement nmspinning or set m.spinning in the newly started M. //go:nowritebarrier func startm(_p_ *p, spinning bool) { lock(&sched.lock) if _p_ == nil { _p_ = pidleget() if _p_ == nil { unlock(&sched.lock) if spinning { // The caller incremented nmspinning, but there are no idle Ps, // so it's okay to just undo the increment and give up. if int32(atomic.Xadd(&sched.nmspinning, -1)) < 0 { throw("startm: negative nmspinning") } } return } } mp := mget() unlock(&sched.lock) if mp == nil { var fn func() if spinning { // The caller incremented nmspinning, so set m.spinning in the new M. fn = mspinning } newm(fn, _p_) return } if mp.spinning { throw("startm: m is spinning") } if mp.nextp != 0 { throw("startm: m has p") } if spinning && !runqempty(_p_) { throw("startm: p has runnable gs") } // The caller incremented nmspinning, so set m.spinning in the new M. mp.spinning = spinning mp.nextp.set(_p_) notewakeup(&mp.park) } // Hands off P from syscall or locked M. // Always runs without a P, so write barriers are not allowed. //go:nowritebarrier func handoffp(_p_ *p) { // handoffp must start an M in any situation where // findrunnable would return a G to run on _p_. // if it has local work, start it straight away if !runqempty(_p_) || sched.runqsize != 0 { startm(_p_, false) return } // if it has GC work, start it straight away if gcBlackenEnabled != 0 && gcMarkWorkAvailable(_p_) { startm(_p_, false) return } // no local work, check that there are no spinning/idle M's, // otherwise our help is not required if atomic.Load(&sched.nmspinning)+atomic.Load(&sched.npidle) == 0 && atomic.Cas(&sched.nmspinning, 0, 1) { // TODO: fast atomic startm(_p_, true) return } lock(&sched.lock) if sched.gcwaiting != 0 { _p_.status = _Pgcstop sched.stopwait-- if sched.stopwait == 0 { notewakeup(&sched.stopnote) } unlock(&sched.lock) return } if _p_.runSafePointFn != 0 && atomic.Cas(&_p_.runSafePointFn, 1, 0) { sched.safePointFn(_p_) sched.safePointWait-- if sched.safePointWait == 0 { notewakeup(&sched.safePointNote) } } if sched.runqsize != 0 { unlock(&sched.lock) startm(_p_, false) return } // If this is the last running P and nobody is polling network, // need to wakeup another M to poll network. if sched.npidle == uint32(gomaxprocs-1) && atomic.Load64(&sched.lastpoll) != 0 { unlock(&sched.lock) startm(_p_, false) return } pidleput(_p_) unlock(&sched.lock) } // Tries to add one more P to execute G's. // Called when a G is made runnable (newproc, ready). func wakep() { // be conservative about spinning threads if !atomic.Cas(&sched.nmspinning, 0, 1) { return } startm(nil, true) } // Stops execution of the current m that is locked to a g until the g is runnable again. // Returns with acquired P. func stoplockedm() { _g_ := getg() if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m { throw("stoplockedm: inconsistent locking") } if _g_.m.p != 0 { // Schedule another M to run this p. _p_ := releasep() handoffp(_p_) } incidlelocked(1) // Wait until another thread schedules lockedg again. notesleep(&_g_.m.park) noteclear(&_g_.m.park) status := readgstatus(_g_.m.lockedg) if status&^_Gscan != _Grunnable { print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n") dumpgstatus(_g_) throw("stoplockedm: not runnable") } acquirep(_g_.m.nextp.ptr()) _g_.m.nextp = 0 } // Schedules the locked m to run the locked gp. // May run during STW, so write barriers are not allowed. //go:nowritebarrier func startlockedm(gp *g) { _g_ := getg() mp := gp.lockedm if mp == _g_.m { throw("startlockedm: locked to me") } if mp.nextp != 0 { throw("startlockedm: m has p") } // directly handoff current P to the locked m incidlelocked(-1) _p_ := releasep() mp.nextp.set(_p_) notewakeup(&mp.park) stopm() } // Stops the current m for stopTheWorld. // Returns when the world is restarted. func gcstopm() { _g_ := getg() if sched.gcwaiting == 0 { throw("gcstopm: not waiting for gc") } if _g_.m.spinning { _g_.m.spinning = false // OK to just drop nmspinning here, // startTheWorld will unpark threads as necessary. if int32(atomic.Xadd(&sched.nmspinning, -1)) < 0 { throw("gcstopm: negative nmspinning") } } _p_ := releasep() lock(&sched.lock) _p_.status = _Pgcstop sched.stopwait-- if sched.stopwait == 0 { notewakeup(&sched.stopnote) } unlock(&sched.lock) stopm() } // Schedules gp to run on the current M. // If inheritTime is true, gp inherits the remaining time in the // current time slice. Otherwise, it starts a new time slice. // Never returns. func execute(gp *g, inheritTime bool) { _g_ := getg() casgstatus(gp, _Grunnable, _Grunning) gp.waitsince = 0 gp.preempt = false gp.stackguard0 = gp.stack.lo + _StackGuard if !inheritTime { _g_.m.p.ptr().schedtick++ } _g_.m.curg = gp gp.m = _g_.m // Check whether the profiler needs to be turned on or off. hz := sched.profilehz if _g_.m.profilehz != hz { resetcpuprofiler(hz) } if trace.enabled { // GoSysExit has to happen when we have a P, but before GoStart. // So we emit it here. if gp.syscallsp != 0 && gp.sysblocktraced { // Since gp.sysblocktraced is true, we must emit an event. // There is a race between the code that initializes sysexitseq // and sysexitticks (in exitsyscall, which runs without a P, // and therefore is not stopped with the rest of the world) // and the code that initializes a new trace. // The recorded sysexitseq and sysexitticks must therefore // be treated as "best effort". If they are valid for this trace, // then great, use them for greater accuracy. // But if they're not valid for this trace, assume that the // trace was started after the actual syscall exit (but before // we actually managed to start the goroutine, aka right now), // and assign a fresh time stamp to keep the log consistent. seq, ts := gp.sysexitseq, gp.sysexitticks if seq == 0 || int64(seq)-int64(trace.seqStart) < 0 { seq, ts = tracestamp() } traceGoSysExit(seq, ts) } traceGoStart() } gogo(&gp.sched) } // Finds a runnable goroutine to execute. // Tries to steal from other P's, get g from global queue, poll network. func findrunnable() (gp *g, inheritTime bool) { _g_ := getg() // The conditions here and in handoffp must agree: if // findrunnable would return a G to run, handoffp must start // an M. top: if sched.gcwaiting != 0 { gcstopm() goto top } if _g_.m.p.ptr().runSafePointFn != 0 { runSafePointFn() } if fingwait && fingwake { if gp := wakefing(); gp != nil { ready(gp, 0) } } // local runq if gp, inheritTime := runqget(_g_.m.p.ptr()); gp != nil { return gp, inheritTime } // global runq if sched.runqsize != 0 { lock(&sched.lock) gp := globrunqget(_g_.m.p.ptr(), 0) unlock(&sched.lock) if gp != nil { return gp, false } } // Poll network. // This netpoll is only an optimization before we resort to stealing. // We can safely skip it if there a thread blocked in netpoll already. // If there is any kind of logical race with that blocked thread // (e.g. it has already returned from netpoll, but does not set lastpoll yet), // this thread will do blocking netpoll below anyway. if netpollinited() && sched.lastpoll != 0 { if gp := netpoll(false); gp != nil { // non-blocking // netpoll returns list of goroutines linked by schedlink. injectglist(gp.schedlink.ptr()) casgstatus(gp, _Gwaiting, _Grunnable) if trace.enabled { traceGoUnpark(gp, 0) } return gp, false } } // If number of spinning M's >= number of busy P's, block. // This is necessary to prevent excessive CPU consumption // when GOMAXPROCS>>1 but the program parallelism is low. if !_g_.m.spinning && 2*atomic.Load(&sched.nmspinning) >= uint32(gomaxprocs)-atomic.Load(&sched.npidle) { // TODO: fast atomic goto stop } if !_g_.m.spinning { _g_.m.spinning = true atomic.Xadd(&sched.nmspinning, 1) } // random steal from other P's for i := 0; i < int(4*gomaxprocs); i++ { if sched.gcwaiting != 0 { goto top } _p_ := allp[fastrand1()%uint32(gomaxprocs)] var gp *g if _p_ == _g_.m.p.ptr() { gp, _ = runqget(_p_) } else { stealRunNextG := i > 2*int(gomaxprocs) // first look for ready queues with more than 1 g gp = runqsteal(_g_.m.p.ptr(), _p_, stealRunNextG) } if gp != nil { return gp, false } } stop: // We have nothing to do. If we're in the GC mark phase, can // safely scan and blacken objects, and have work to do, run // idle-time marking rather than give up the P. if _p_ := _g_.m.p.ptr(); gcBlackenEnabled != 0 && _p_.gcBgMarkWorker != 0 && gcMarkWorkAvailable(_p_) { _p_.gcMarkWorkerMode = gcMarkWorkerIdleMode gp := _p_.gcBgMarkWorker.ptr() casgstatus(gp, _Gwaiting, _Grunnable) if trace.enabled { traceGoUnpark(gp, 0) } return gp, false } // return P and block lock(&sched.lock) if sched.gcwaiting != 0 || _g_.m.p.ptr().runSafePointFn != 0 { unlock(&sched.lock) goto top } if sched.runqsize != 0 { gp := globrunqget(_g_.m.p.ptr(), 0) unlock(&sched.lock) return gp, false } _p_ := releasep() pidleput(_p_) unlock(&sched.lock) // Delicate dance: thread transitions from spinning to non-spinning state, // potentially concurrently with submission of new goroutines. We must // drop nmspinning first and then check all per-P queues again (with // #StoreLoad memory barrier in between). If we do it the other way around, // another thread can submit a goroutine after we've checked all run queues // but before we drop nmspinning; as the result nobody will unpark a thread // to run the goroutine. // If we discover new work below, we need to restore m.spinning as a signal // for resetspinning to unpark a new worker thread (because there can be more // than one starving goroutine). However, if after discovering new work // we also observe no idle Ps, it is OK to just park the current thread: // the system is fully loaded so no spinning threads are required. // Also see "Worker thread parking/unparking" comment at the top of the file. wasSpinning := _g_.m.spinning if _g_.m.spinning { _g_.m.spinning = false if int32(atomic.Xadd(&sched.nmspinning, -1)) < 0 { throw("findrunnable: negative nmspinning") } } // check all runqueues once again for i := 0; i < int(gomaxprocs); i++ { _p_ := allp[i] if _p_ != nil && !runqempty(_p_) { lock(&sched.lock) _p_ = pidleget() unlock(&sched.lock) if _p_ != nil { acquirep(_p_) if wasSpinning { _g_.m.spinning = true atomic.Xadd(&sched.nmspinning, 1) } goto top } break } } // poll network if netpollinited() && atomic.Xchg64(&sched.lastpoll, 0) != 0 { if _g_.m.p != 0 { throw("findrunnable: netpoll with p") } if _g_.m.spinning { throw("findrunnable: netpoll with spinning") } gp := netpoll(true) // block until new work is available atomic.Store64(&sched.lastpoll, uint64(nanotime())) if gp != nil { lock(&sched.lock) _p_ = pidleget() unlock(&sched.lock) if _p_ != nil { acquirep(_p_) injectglist(gp.schedlink.ptr()) casgstatus(gp, _Gwaiting, _Grunnable) if trace.enabled { traceGoUnpark(gp, 0) } return gp, false } injectglist(gp) } } stopm() goto top } func resetspinning() { _g_ := getg() if !_g_.m.spinning { throw("resetspinning: not a spinning m") } _g_.m.spinning = false nmspinning := atomic.Xadd(&sched.nmspinning, -1) if int32(nmspinning) < 0 { throw("findrunnable: negative nmspinning") } // M wakeup policy is deliberately somewhat conservative, so check if we // need to wakeup another P here. See "Worker thread parking/unparking" // comment at the top of the file for details. if nmspinning == 0 && atomic.Load(&sched.npidle) > 0 { wakep() } } // Injects the list of runnable G's into the scheduler. // Can run concurrently with GC. func injectglist(glist *g) { if glist == nil { return } if trace.enabled { for gp := glist; gp != nil; gp = gp.schedlink.ptr() { traceGoUnpark(gp, 0) } } lock(&sched.lock) var n int for n = 0; glist != nil; n++ { gp := glist glist = gp.schedlink.ptr() casgstatus(gp, _Gwaiting, _Grunnable) globrunqput(gp) } unlock(&sched.lock) for ; n != 0 && sched.npidle != 0; n-- { startm(nil, false) } } // One round of scheduler: find a runnable goroutine and execute it. // Never returns. func schedule() { _g_ := getg() if _g_.m.locks != 0 { throw("schedule: holding locks") } if _g_.m.lockedg != nil { stoplockedm() execute(_g_.m.lockedg, false) // Never returns. } top: if sched.gcwaiting != 0 { gcstopm() goto top } if _g_.m.p.ptr().runSafePointFn != 0 { runSafePointFn() } var gp *g var inheritTime bool if trace.enabled || trace.shutdown { gp = traceReader() if gp != nil { casgstatus(gp, _Gwaiting, _Grunnable) traceGoUnpark(gp, 0) } } if gp == nil && gcBlackenEnabled != 0 { gp = gcController.findRunnableGCWorker(_g_.m.p.ptr()) } if gp == nil { // Check the global runnable queue once in a while to ensure fairness. // Otherwise two goroutines can completely occupy the local runqueue // by constantly respawning each other. if _g_.m.p.ptr().schedtick%61 == 0 && sched.runqsize > 0 { lock(&sched.lock) gp = globrunqget(_g_.m.p.ptr(), 1) unlock(&sched.lock) } } if gp == nil { gp, inheritTime = runqget(_g_.m.p.ptr()) if gp != nil && _g_.m.spinning { throw("schedule: spinning with local work") } } if gp == nil { gp, inheritTime = findrunnable() // blocks until work is available } // This thread is going to run a goroutine and is not spinning anymore, // so if it was marked as spinning we need to reset it now and potentially // start a new spinning M. if _g_.m.spinning { resetspinning() } if gp.lockedm != nil { // Hands off own p to the locked m, // then blocks waiting for a new p. startlockedm(gp) goto top } execute(gp, inheritTime) } // dropg removes the association between m and the current goroutine m->curg (gp for short). // Typically a caller sets gp's status away from Grunning and then // immediately calls dropg to finish the job. The caller is also responsible // for arranging that gp will be restarted using ready at an // appropriate time. After calling dropg and arranging for gp to be // readied later, the caller can do other work but eventually should // call schedule to restart the scheduling of goroutines on this m. func dropg() { _g_ := getg() if _g_.m.lockedg == nil { _g_.m.curg.m = nil _g_.m.curg = nil } } func parkunlock_c(gp *g, lock unsafe.Pointer) bool { unlock((*mutex)(lock)) return true } // park continuation on g0. func park_m(gp *g) { _g_ := getg() if trace.enabled { traceGoPark(_g_.m.waittraceev, _g_.m.waittraceskip, gp) } casgstatus(gp, _Grunning, _Gwaiting) dropg() if _g_.m.waitunlockf != nil { fn := *(*func(*g, unsafe.Pointer) bool)(unsafe.Pointer(&_g_.m.waitunlockf)) ok := fn(gp, _g_.m.waitlock) _g_.m.waitunlockf = nil _g_.m.waitlock = nil if !ok { if trace.enabled { traceGoUnpark(gp, 2) } casgstatus(gp, _Gwaiting, _Grunnable) execute(gp, true) // Schedule it back, never returns. } } schedule() } func goschedImpl(gp *g) { status := readgstatus(gp) if status&^_Gscan != _Grunning { dumpgstatus(gp) throw("bad g status") } casgstatus(gp, _Grunning, _Grunnable) dropg() lock(&sched.lock) globrunqput(gp) unlock(&sched.lock) schedule() } // Gosched continuation on g0. func gosched_m(gp *g) { if trace.enabled { traceGoSched() } goschedImpl(gp) } func gopreempt_m(gp *g) { if trace.enabled { traceGoPreempt() } goschedImpl(gp) } // Finishes execution of the current goroutine. func goexit1() { if raceenabled { racegoend() } if trace.enabled { traceGoEnd() } mcall(goexit0) } // goexit continuation on g0. func goexit0(gp *g) { _g_ := getg() casgstatus(gp, _Grunning, _Gdead) if isSystemGoroutine(gp) { atomic.Xadd(&sched.ngsys, -1) } gp.m = nil gp.lockedm = nil _g_.m.lockedg = nil gp.paniconfault = false gp._defer = nil // should be true already but just in case. gp._panic = nil // non-nil for Goexit during panic. points at stack-allocated data. gp.writebuf = nil gp.waitreason = "" gp.param = nil dropg() if _g_.m.locked&^_LockExternal != 0 { print("invalid m->locked = ", _g_.m.locked, "\n") throw("internal lockOSThread error") } _g_.m.locked = 0 gfput(_g_.m.p.ptr(), gp) schedule() } //go:nosplit //go:nowritebarrier func save(pc, sp uintptr) { _g_ := getg() _g_.sched.pc = pc _g_.sched.sp = sp _g_.sched.lr = 0 _g_.sched.ret = 0 _g_.sched.ctxt = nil _g_.sched.g = guintptr(unsafe.Pointer(_g_)) } // The goroutine g is about to enter a system call. // Record that it's not using the cpu anymore. // This is called only from the go syscall library and cgocall, // not from the low-level system calls used by the runtime. // // Entersyscall cannot split the stack: the gosave must // make g->sched refer to the caller's stack segment, because // entersyscall is going to return immediately after. // // Nothing entersyscall calls can split the stack either. // We cannot safely move the stack during an active call to syscall, // because we do not know which of the uintptr arguments are // really pointers (back into the stack). // In practice, this means that we make the fast path run through // entersyscall doing no-split things, and the slow path has to use systemstack // to run bigger things on the system stack. // // reentersyscall is the entry point used by cgo callbacks, where explicitly // saved SP and PC are restored. This is needed when exitsyscall will be called // from a function further up in the call stack than the parent, as g->syscallsp // must always point to a valid stack frame. entersyscall below is the normal // entry point for syscalls, which obtains the SP and PC from the caller. // // Syscall tracing: // At the start of a syscall we emit traceGoSysCall to capture the stack trace. // If the syscall does not block, that is it, we do not emit any other events. // If the syscall blocks (that is, P is retaken), retaker emits traceGoSysBlock; // when syscall returns we emit traceGoSysExit and when the goroutine starts running // (potentially instantly, if exitsyscallfast returns true) we emit traceGoStart. // To ensure that traceGoSysExit is emitted strictly after traceGoSysBlock, // we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.ptr().syscalltick), // whoever emits traceGoSysBlock increments p.syscalltick afterwards; // and we wait for the increment before emitting traceGoSysExit. // Note that the increment is done even if tracing is not enabled, // because tracing can be enabled in the middle of syscall. We don't want the wait to hang. // //go:nosplit func reentersyscall(pc, sp uintptr) { _g_ := getg() // Disable preemption because during this function g is in Gsyscall status, // but can have inconsistent g->sched, do not let GC observe it. _g_.m.locks++ // Entersyscall must not call any function that might split/grow the stack. // (See details in comment above.) // Catch calls that might, by replacing the stack guard with something that // will trip any stack check and leaving a flag to tell newstack to die. _g_.stackguard0 = stackPreempt _g_.throwsplit = true // Leave SP around for GC and traceback. save(pc, sp) _g_.syscallsp = sp _g_.syscallpc = pc casgstatus(_g_, _Grunning, _Gsyscall) if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { systemstack(func() { print("entersyscall inconsistent ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n") throw("entersyscall") }) } if trace.enabled { systemstack(traceGoSysCall) // systemstack itself clobbers g.sched.{pc,sp} and we might // need them later when the G is genuinely blocked in a // syscall save(pc, sp) } if atomic.Load(&sched.sysmonwait) != 0 { // TODO: fast atomic systemstack(entersyscall_sysmon) save(pc, sp) } if _g_.m.p.ptr().runSafePointFn != 0 { // runSafePointFn may stack split if run on this stack systemstack(runSafePointFn) save(pc, sp) } _g_.m.syscalltick = _g_.m.p.ptr().syscalltick _g_.sysblocktraced = true _g_.m.mcache = nil _g_.m.p.ptr().m = 0 atomic.Store(&_g_.m.p.ptr().status, _Psyscall) if sched.gcwaiting != 0 { systemstack(entersyscall_gcwait) save(pc, sp) } // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched). // We set _StackGuard to StackPreempt so that first split stack check calls morestack. // Morestack detects this case and throws. _g_.stackguard0 = stackPreempt _g_.m.locks-- } // Standard syscall entry used by the go syscall library and normal cgo calls. //go:nosplit func entersyscall(dummy int32) { reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) } func entersyscall_sysmon() { lock(&sched.lock) if atomic.Load(&sched.sysmonwait) != 0 { atomic.Store(&sched.sysmonwait, 0) notewakeup(&sched.sysmonnote) } unlock(&sched.lock) } func entersyscall_gcwait() { _g_ := getg() _p_ := _g_.m.p.ptr() lock(&sched.lock) if sched.stopwait > 0 && atomic.Cas(&_p_.status, _Psyscall, _Pgcstop) { if trace.enabled { traceGoSysBlock(_p_) traceProcStop(_p_) } _p_.syscalltick++ if sched.stopwait--; sched.stopwait == 0 { notewakeup(&sched.stopnote) } } unlock(&sched.lock) } // The same as entersyscall(), but with a hint that the syscall is blocking. //go:nosplit func entersyscallblock(dummy int32) { _g_ := getg() _g_.m.locks++ // see comment in entersyscall _g_.throwsplit = true _g_.stackguard0 = stackPreempt // see comment in entersyscall _g_.m.syscalltick = _g_.m.p.ptr().syscalltick _g_.sysblocktraced = true _g_.m.p.ptr().syscalltick++ // Leave SP around for GC and traceback. pc := getcallerpc(unsafe.Pointer(&dummy)) sp := getcallersp(unsafe.Pointer(&dummy)) save(pc, sp) _g_.syscallsp = _g_.sched.sp _g_.syscallpc = _g_.sched.pc if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { sp1 := sp sp2 := _g_.sched.sp sp3 := _g_.syscallsp systemstack(func() { print("entersyscallblock inconsistent ", hex(sp1), " ", hex(sp2), " ", hex(sp3), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n") throw("entersyscallblock") }) } casgstatus(_g_, _Grunning, _Gsyscall) if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { systemstack(func() { print("entersyscallblock inconsistent ", hex(sp), " ", hex(_g_.sched.sp), " ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n") throw("entersyscallblock") }) } systemstack(entersyscallblock_handoff) // Resave for traceback during blocked call. save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) _g_.m.locks-- } func entersyscallblock_handoff() { if trace.enabled { traceGoSysCall() traceGoSysBlock(getg().m.p.ptr()) } handoffp(releasep()) } // The goroutine g exited its system call. // Arrange for it to run on a cpu again. // This is called only from the go syscall library, not // from the low-level system calls used by the //go:nosplit func exitsyscall(dummy int32) { _g_ := getg() _g_.m.locks++ // see comment in entersyscall if getcallersp(unsafe.Pointer(&dummy)) > _g_.syscallsp { throw("exitsyscall: syscall frame is no longer valid") } _g_.waitsince = 0 oldp := _g_.m.p.ptr() if exitsyscallfast() { if _g_.m.mcache == nil { throw("lost mcache") } if trace.enabled { if oldp != _g_.m.p.ptr() || _g_.m.syscalltick != _g_.m.p.ptr().syscalltick { systemstack(traceGoStart) } } // There's a cpu for us, so we can run. _g_.m.p.ptr().syscalltick++ // We need to cas the status and scan before resuming... casgstatus(_g_, _Gsyscall, _Grunning) // Garbage collector isn't running (since we are), // so okay to clear syscallsp. _g_.syscallsp = 0 _g_.m.locks-- if _g_.preempt { // restore the preemption request in case we've cleared it in newstack _g_.stackguard0 = stackPreempt } else { // otherwise restore the real _StackGuard, we've spoiled it in entersyscall/entersyscallblock _g_.stackguard0 = _g_.stack.lo + _StackGuard } _g_.throwsplit = false return } _g_.sysexitticks = 0 _g_.sysexitseq = 0 if trace.enabled { // Wait till traceGoSysBlock event is emitted. // This ensures consistency of the trace (the goroutine is started after it is blocked). for oldp != nil && oldp.syscalltick == _g_.m.syscalltick { osyield() } // We can't trace syscall exit right now because we don't have a P. // Tracing code can invoke write barriers that cannot run without a P. // So instead we remember the syscall exit time and emit the event // in execute when we have a P. _g_.sysexitseq, _g_.sysexitticks = tracestamp() } _g_.m.locks-- // Call the scheduler. mcall(exitsyscall0) if _g_.m.mcache == nil { throw("lost mcache") } // Scheduler returned, so we're allowed to run now. // Delete the syscallsp information that we left for // the garbage collector during the system call. // Must wait until now because until gosched returns // we don't know for sure that the garbage collector // is not running. _g_.syscallsp = 0 _g_.m.p.ptr().syscalltick++ _g_.throwsplit = false } //go:nosplit func exitsyscallfast() bool { _g_ := getg() // Freezetheworld sets stopwait but does not retake P's. if sched.stopwait == freezeStopWait { _g_.m.mcache = nil _g_.m.p = 0 return false } // Try to re-acquire the last P. if _g_.m.p != 0 && _g_.m.p.ptr().status == _Psyscall && atomic.Cas(&_g_.m.p.ptr().status, _Psyscall, _Prunning) { // There's a cpu for us, so we can run. _g_.m.mcache = _g_.m.p.ptr().mcache _g_.m.p.ptr().m.set(_g_.m) if _g_.m.syscalltick != _g_.m.p.ptr().syscalltick { if trace.enabled { // The p was retaken and then enter into syscall again (since _g_.m.syscalltick has changed). // traceGoSysBlock for this syscall was already emitted, // but here we effectively retake the p from the new syscall running on the same p. systemstack(func() { // Denote blocking of the new syscall. traceGoSysBlock(_g_.m.p.ptr()) // Denote completion of the current syscall. traceGoSysExit(tracestamp()) }) } _g_.m.p.ptr().syscalltick++ } return true } // Try to get any other idle P. oldp := _g_.m.p.ptr() _g_.m.mcache = nil _g_.m.p = 0 if sched.pidle != 0 { var ok bool systemstack(func() { ok = exitsyscallfast_pidle() if ok && trace.enabled { if oldp != nil { // Wait till traceGoSysBlock event is emitted. // This ensures consistency of the trace (the goroutine is started after it is blocked). for oldp.syscalltick == _g_.m.syscalltick { osyield() } } traceGoSysExit(tracestamp()) } }) if ok { return true } } return false } func exitsyscallfast_pidle() bool { lock(&sched.lock) _p_ := pidleget() if _p_ != nil && atomic.Load(&sched.sysmonwait) != 0 { atomic.Store(&sched.sysmonwait, 0) notewakeup(&sched.sysmonnote) } unlock(&sched.lock) if _p_ != nil { acquirep(_p_) return true } return false } // exitsyscall slow path on g0. // Failed to acquire P, enqueue gp as runnable. func exitsyscall0(gp *g) { _g_ := getg() casgstatus(gp, _Gsyscall, _Grunnable) dropg() lock(&sched.lock) _p_ := pidleget() if _p_ == nil { globrunqput(gp) } else if atomic.Load(&sched.sysmonwait) != 0 { atomic.Store(&sched.sysmonwait, 0) notewakeup(&sched.sysmonnote) } unlock(&sched.lock) if _p_ != nil { acquirep(_p_) execute(gp, false) // Never returns. } if _g_.m.lockedg != nil { // Wait until another thread schedules gp and so m again. stoplockedm() execute(gp, false) // Never returns. } stopm() schedule() // Never returns. } func beforefork() { gp := getg().m.curg // Fork can hang if preempted with signals frequently enough (see issue 5517). // Ensure that we stay on the same M where we disable profiling. gp.m.locks++ if gp.m.profilehz != 0 { resetcpuprofiler(0) } // This function is called before fork in syscall package. // Code between fork and exec must not allocate memory nor even try to grow stack. // Here we spoil g->_StackGuard to reliably detect any attempts to grow stack. // runtime_AfterFork will undo this in parent process, but not in child. gp.stackguard0 = stackFork } // Called from syscall package before fork. //go:linkname syscall_runtime_BeforeFork syscall.runtime_BeforeFork //go:nosplit func syscall_runtime_BeforeFork() { systemstack(beforefork) } func afterfork() { gp := getg().m.curg // See the comment in beforefork. gp.stackguard0 = gp.stack.lo + _StackGuard hz := sched.profilehz if hz != 0 { resetcpuprofiler(hz) } gp.m.locks-- } // Called from syscall package after fork in parent. //go:linkname syscall_runtime_AfterFork syscall.runtime_AfterFork //go:nosplit func syscall_runtime_AfterFork() { systemstack(afterfork) } // Allocate a new g, with a stack big enough for stacksize bytes. func malg(stacksize int32) *g { newg := new(g) if stacksize >= 0 { stacksize = round2(_StackSystem + stacksize) systemstack(func() { newg.stack, newg.stkbar = stackalloc(uint32(stacksize)) }) newg.stackguard0 = newg.stack.lo + _StackGuard newg.stackguard1 = ^uintptr(0) newg.stackAlloc = uintptr(stacksize) } return newg } // Create a new g running fn with siz bytes of arguments. // Put it on the queue of g's waiting to run. // The compiler turns a go statement into a call to this. // Cannot split the stack because it assumes that the arguments // are available sequentially after &fn; they would not be // copied if a stack split occurred. //go:nosplit func newproc(siz int32, fn *funcval) { argp := add(unsafe.Pointer(&fn), sys.PtrSize) pc := getcallerpc(unsafe.Pointer(&siz)) systemstack(func() { newproc1(fn, (*uint8)(argp), siz, 0, pc) }) } // Create a new g running fn with narg bytes of arguments starting // at argp and returning nret bytes of results. callerpc is the // address of the go statement that created this. The new g is put // on the queue of g's waiting to run. func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr) *g { _g_ := getg() if fn == nil { _g_.m.throwing = -1 // do not dump full stacks throw("go of nil func value") } _g_.m.locks++ // disable preemption because it can be holding p in a local var siz := narg + nret siz = (siz + 7) &^ 7 // We could allocate a larger initial stack if necessary. // Not worth it: this is almost always an error. // 4*sizeof(uintreg): extra space added below // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall). if siz >= _StackMin-4*sys.RegSize-sys.RegSize { throw("newproc: function arguments too large for new goroutine") } _p_ := _g_.m.p.ptr() newg := gfget(_p_) if newg == nil { newg = malg(_StackMin) casgstatus(newg, _Gidle, _Gdead) allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack. } if newg.stack.hi == 0 { throw("newproc1: newg missing stack") } if readgstatus(newg) != _Gdead { throw("newproc1: new g is not Gdead") } totalSize := 4*sys.RegSize + uintptr(siz) + sys.MinFrameSize // extra space in case of reads slightly beyond frame totalSize += -totalSize & (sys.SpAlign - 1) // align to spAlign sp := newg.stack.hi - totalSize spArg := sp if usesLR { // caller's LR *(*unsafe.Pointer)(unsafe.Pointer(sp)) = nil prepGoExitFrame(sp) spArg += sys.MinFrameSize } memmove(unsafe.Pointer(spArg), unsafe.Pointer(argp), uintptr(narg)) memclr(unsafe.Pointer(&newg.sched), unsafe.Sizeof(newg.sched)) newg.sched.sp = sp newg.stktopsp = sp newg.sched.pc = funcPC(goexit) + sys.PCQuantum // +PCQuantum so that previous instruction is in same function newg.sched.g = guintptr(unsafe.Pointer(newg)) gostartcallfn(&newg.sched, fn) newg.gopc = callerpc newg.startpc = fn.fn if isSystemGoroutine(newg) { atomic.Xadd(&sched.ngsys, +1) } casgstatus(newg, _Gdead, _Grunnable) if _p_.goidcache == _p_.goidcacheend { // Sched.goidgen is the last allocated id, // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch]. // At startup sched.goidgen=0, so main goroutine receives goid=1. _p_.goidcache = atomic.Xadd64(&sched.goidgen, _GoidCacheBatch) _p_.goidcache -= _GoidCacheBatch - 1 _p_.goidcacheend = _p_.goidcache + _GoidCacheBatch } newg.goid = int64(_p_.goidcache) _p_.goidcache++ if raceenabled { newg.racectx = racegostart(callerpc) } if trace.enabled { traceGoCreate(newg, newg.startpc) } runqput(_p_, newg, true) if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 && unsafe.Pointer(fn.fn) != unsafe.Pointer(funcPC(main)) { // TODO: fast atomic wakep() } _g_.m.locks-- if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack _g_.stackguard0 = stackPreempt } return newg } // Put on gfree list. // If local list is too long, transfer a batch to the global list. func gfput(_p_ *p, gp *g) { if readgstatus(gp) != _Gdead { throw("gfput: bad status (not Gdead)") } stksize := gp.stackAlloc if stksize != _FixedStack { // non-standard stack size - free it. stackfree(gp.stack, gp.stackAlloc) gp.stack.lo = 0 gp.stack.hi = 0 gp.stackguard0 = 0 gp.stkbar = nil gp.stkbarPos = 0 } else { // Reset stack barriers. gp.stkbar = gp.stkbar[:0] gp.stkbarPos = 0 } gp.schedlink.set(_p_.gfree) _p_.gfree = gp _p_.gfreecnt++ if _p_.gfreecnt >= 64 { lock(&sched.gflock) for _p_.gfreecnt >= 32 { _p_.gfreecnt-- gp = _p_.gfree _p_.gfree = gp.schedlink.ptr() gp.schedlink.set(sched.gfree) sched.gfree = gp sched.ngfree++ } unlock(&sched.gflock) } } // Get from gfree list. // If local list is empty, grab a batch from global list. func gfget(_p_ *p) *g { retry: gp := _p_.gfree if gp == nil && sched.gfree != nil { lock(&sched.gflock) for _p_.gfreecnt < 32 && sched.gfree != nil { _p_.gfreecnt++ gp = sched.gfree sched.gfree = gp.schedlink.ptr() sched.ngfree-- gp.schedlink.set(_p_.gfree) _p_.gfree = gp } unlock(&sched.gflock) goto retry } if gp != nil { _p_.gfree = gp.schedlink.ptr() _p_.gfreecnt-- if gp.stack.lo == 0 { // Stack was deallocated in gfput. Allocate a new one. systemstack(func() { gp.stack, gp.stkbar = stackalloc(_FixedStack) }) gp.stackguard0 = gp.stack.lo + _StackGuard gp.stackAlloc = _FixedStack } else { if raceenabled { racemalloc(unsafe.Pointer(gp.stack.lo), gp.stackAlloc) } if msanenabled { msanmalloc(unsafe.Pointer(gp.stack.lo), gp.stackAlloc) } } } return gp } // Purge all cached G's from gfree list to the global list. func gfpurge(_p_ *p) { lock(&sched.gflock) for _p_.gfreecnt != 0 { _p_.gfreecnt-- gp := _p_.gfree _p_.gfree = gp.schedlink.ptr() gp.schedlink.set(sched.gfree) sched.gfree = gp sched.ngfree++ } unlock(&sched.gflock) } // Breakpoint executes a breakpoint trap. func Breakpoint() { breakpoint() } // dolockOSThread is called by LockOSThread and lockOSThread below // after they modify m.locked. Do not allow preemption during this call, // or else the m might be different in this function than in the caller. //go:nosplit func dolockOSThread() { _g_ := getg() _g_.m.lockedg = _g_ _g_.lockedm = _g_.m } //go:nosplit // LockOSThread wires the calling goroutine to its current operating system thread. // Until the calling goroutine exits or calls UnlockOSThread, it will always // execute in that thread, and no other goroutine can. func LockOSThread() { getg().m.locked |= _LockExternal dolockOSThread() } //go:nosplit func lockOSThread() { getg().m.locked += _LockInternal dolockOSThread() } // dounlockOSThread is called by UnlockOSThread and unlockOSThread below // after they update m->locked. Do not allow preemption during this call, // or else the m might be in different in this function than in the caller. //go:nosplit func dounlockOSThread() { _g_ := getg() if _g_.m.locked != 0 { return } _g_.m.lockedg = nil _g_.lockedm = nil } //go:nosplit // UnlockOSThread unwires the calling goroutine from its fixed operating system thread. // If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op. func UnlockOSThread() { getg().m.locked &^= _LockExternal dounlockOSThread() } //go:nosplit func unlockOSThread() { _g_ := getg() if _g_.m.locked < _LockInternal { systemstack(badunlockosthread) } _g_.m.locked -= _LockInternal dounlockOSThread() } func badunlockosthread() { throw("runtime: internal error: misuse of lockOSThread/unlockOSThread") } func gcount() int32 { n := int32(allglen) - sched.ngfree - int32(atomic.Load(&sched.ngsys)) for i := 0; ; i++ { _p_ := allp[i] if _p_ == nil { break } n -= _p_.gfreecnt } // All these variables can be changed concurrently, so the result can be inconsistent. // But at least the current goroutine is running. if n < 1 { n = 1 } return n } func mcount() int32 { return sched.mcount } var prof struct { lock uint32 hz int32 } func _System() { _System() } func _ExternalCode() { _ExternalCode() } func _GC() { _GC() } // Called if we receive a SIGPROF signal. func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { if prof.hz == 0 { return } // Profiling runs concurrently with GC, so it must not allocate. mp.mallocing++ // Define that a "user g" is a user-created goroutine, and a "system g" // is one that is m->g0 or m->gsignal. // // We might be interrupted for profiling halfway through a // goroutine switch. The switch involves updating three (or four) values: // g, PC, SP, and (on arm) LR. The PC must be the last to be updated, // because once it gets updated the new g is running. // // When switching from a user g to a system g, LR is not considered live, // so the update only affects g, SP, and PC. Since PC must be last, there // the possible partial transitions in ordinary execution are (1) g alone is updated, // (2) both g and SP are updated, and (3) SP alone is updated. // If SP or g alone is updated, we can detect the partial transition by checking // whether the SP is within g's stack bounds. (We could also require that SP // be changed only after g, but the stack bounds check is needed by other // cases, so there is no need to impose an additional requirement.) // // There is one exceptional transition to a system g, not in ordinary execution. // When a signal arrives, the operating system starts the signal handler running // with an updated PC and SP. The g is updated last, at the beginning of the // handler. There are two reasons this is okay. First, until g is updated the // g and SP do not match, so the stack bounds check detects the partial transition. // Second, signal handlers currently run with signals disabled, so a profiling // signal cannot arrive during the handler. // // When switching from a system g to a user g, there are three possibilities. // // First, it may be that the g switch has no PC update, because the SP // either corresponds to a user g throughout (as in asmcgocall) // or because it has been arranged to look like a user g frame // (as in cgocallback_gofunc). In this case, since the entire // transition is a g+SP update, a partial transition updating just one of // those will be detected by the stack bounds check. // // Second, when returning from a signal handler, the PC and SP updates // are performed by the operating system in an atomic update, so the g // update must be done before them. The stack bounds check detects // the partial transition here, and (again) signal handlers run with signals // disabled, so a profiling signal cannot arrive then anyway. // // Third, the common case: it may be that the switch updates g, SP, and PC // separately. If the PC is within any of the functions that does this, // we don't ask for a traceback. C.F. the function setsSP for more about this. // // There is another apparently viable approach, recorded here in case // the "PC within setsSP function" check turns out not to be usable. // It would be possible to delay the update of either g or SP until immediately // before the PC update instruction. Then, because of the stack bounds check, // the only problematic interrupt point is just before that PC update instruction, // and the sigprof handler can detect that instruction and simulate stepping past // it in order to reach a consistent state. On ARM, the update of g must be made // in two places (in R10 and also in a TLS slot), so the delayed update would // need to be the SP update. The sigprof handler must read the instruction at // the current PC and if it was the known instruction (for example, JMP BX or // MOV R2, PC), use that other register in place of the PC value. // The biggest drawback to this solution is that it requires that we can tell // whether it's safe to read from the memory pointed at by PC. // In a correct program, we can test PC == nil and otherwise read, // but if a profiling signal happens at the instant that a program executes // a bad jump (before the program manages to handle the resulting fault) // the profiling handler could fault trying to read nonexistent memory. // // To recap, there are no constraints on the assembly being used for the // transition. We simply require that g and SP match and that the PC is not // in gogo. traceback := true if gp == nil || sp < gp.stack.lo || gp.stack.hi < sp || setsSP(pc) { traceback = false } var stk [maxCPUProfStack]uintptr var haveStackLock *g n := 0 if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 { // Cgo, we can't unwind and symbolize arbitrary C code, // so instead collect Go stack that leads to the cgo call. // This is especially important on windows, since all syscalls are cgo calls. if gcTryLockStackBarriers(mp.curg) { haveStackLock = mp.curg n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0) } } else if traceback { var flags uint = _TraceTrap if gp.m.curg != nil && gcTryLockStackBarriers(gp.m.curg) { // It's safe to traceback the user stack. haveStackLock = gp.m.curg flags |= _TraceJumpStack } // Traceback is safe if we're on the system stack (if // necessary, flags will stop it before switching to // the user stack), or if we locked the user stack. if gp != gp.m.curg || haveStackLock != nil { n = gentraceback(pc, sp, lr, gp, 0, &stk[0], len(stk), nil, nil, flags) } } if haveStackLock != nil { gcUnlockStackBarriers(haveStackLock) } if n <= 0 { // Normal traceback is impossible or has failed. // See if it falls into several common cases. n = 0 if GOOS == "windows" && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 { // Libcall, i.e. runtime syscall on windows. // Collect Go stack that leads to the call. if gcTryLockStackBarriers(mp.libcallg.ptr()) { n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, 0) gcUnlockStackBarriers(mp.libcallg.ptr()) } } if n == 0 { // If all of the above has failed, account it against abstract "System" or "GC". n = 2 // "ExternalCode" is better than "etext". if pc > firstmoduledata.etext { pc = funcPC(_ExternalCode) + sys.PCQuantum } stk[0] = pc if mp.preemptoff != "" || mp.helpgc != 0 { stk[1] = funcPC(_GC) + sys.PCQuantum } else { stk[1] = funcPC(_System) + sys.PCQuantum } } } if prof.hz != 0 { // Simple cas-lock to coordinate with setcpuprofilerate. for !atomic.Cas(&prof.lock, 0, 1) { osyield() } if prof.hz != 0 { cpuprof.add(stk[:n]) } atomic.Store(&prof.lock, 0) } mp.mallocing-- } // Reports whether a function will set the SP // to an absolute value. Important that // we don't traceback when these are at the bottom // of the stack since we can't be sure that we will // find the caller. // // If the function is not on the bottom of the stack // we assume that it will have set it up so that traceback will be consistent, // either by being a traceback terminating function // or putting one on the stack at the right offset. func setsSP(pc uintptr) bool { f := findfunc(pc) if f == nil { // couldn't find the function for this PC, // so assume the worst and stop traceback return true } switch f.entry { case gogoPC, systemstackPC, mcallPC, morestackPC: return true } return false } // Arrange to call fn with a traceback hz times a second. func setcpuprofilerate_m(hz int32) { // Force sane arguments. if hz < 0 { hz = 0 } // Disable preemption, otherwise we can be rescheduled to another thread // that has profiling enabled. _g_ := getg() _g_.m.locks++ // Stop profiler on this thread so that it is safe to lock prof. // if a profiling signal came in while we had prof locked, // it would deadlock. resetcpuprofiler(0) for !atomic.Cas(&prof.lock, 0, 1) { osyield() } prof.hz = hz atomic.Store(&prof.lock, 0) lock(&sched.lock) sched.profilehz = hz unlock(&sched.lock) if hz != 0 { resetcpuprofiler(hz) } _g_.m.locks-- } // Change number of processors. The world is stopped, sched is locked. // gcworkbufs are not being modified by either the GC or // the write barrier code. // Returns list of Ps with local work, they need to be scheduled by the caller. func procresize(nprocs int32) *p { old := gomaxprocs if old < 0 || old > _MaxGomaxprocs || nprocs <= 0 || nprocs > _MaxGomaxprocs { throw("procresize: invalid arg") } if trace.enabled { traceGomaxprocs(nprocs) } // update statistics now := nanotime() if sched.procresizetime != 0 { sched.totaltime += int64(old) * (now - sched.procresizetime) } sched.procresizetime = now // initialize new P's for i := int32(0); i < nprocs; i++ { pp := allp[i] if pp == nil { pp = new(p) pp.id = i pp.status = _Pgcstop pp.sudogcache = pp.sudogbuf[:0] for i := range pp.deferpool { pp.deferpool[i] = pp.deferpoolbuf[i][:0] } atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(pp)) } if pp.mcache == nil { if old == 0 && i == 0 { if getg().m.mcache == nil { throw("missing mcache?") } pp.mcache = getg().m.mcache // bootstrap } else { pp.mcache = allocmcache() } } } // free unused P's for i := nprocs; i < old; i++ { p := allp[i] if trace.enabled { if p == getg().m.p.ptr() { // moving to p[0], pretend that we were descheduled // and then scheduled again to keep the trace sane. traceGoSched() traceProcStop(p) } } // move all runnable goroutines to the global queue for p.runqhead != p.runqtail { // pop from tail of local queue p.runqtail-- gp := p.runq[p.runqtail%uint32(len(p.runq))].ptr() // push onto head of global queue globrunqputhead(gp) } if p.runnext != 0 { globrunqputhead(p.runnext.ptr()) p.runnext = 0 } // if there's a background worker, make it runnable and put // it on the global queue so it can clean itself up if gp := p.gcBgMarkWorker.ptr(); gp != nil { casgstatus(gp, _Gwaiting, _Grunnable) if trace.enabled { traceGoUnpark(gp, 0) } globrunqput(gp) // This assignment doesn't race because the // world is stopped. p.gcBgMarkWorker.set(nil) } for i := range p.sudogbuf { p.sudogbuf[i] = nil } p.sudogcache = p.sudogbuf[:0] for i := range p.deferpool { for j := range p.deferpoolbuf[i] { p.deferpoolbuf[i][j] = nil } p.deferpool[i] = p.deferpoolbuf[i][:0] } freemcache(p.mcache) p.mcache = nil gfpurge(p) traceProcFree(p) p.status = _Pdead // can't free P itself because it can be referenced by an M in syscall } _g_ := getg() if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs { // continue to use the current P _g_.m.p.ptr().status = _Prunning } else { // release the current P and acquire allp[0] if _g_.m.p != 0 { _g_.m.p.ptr().m = 0 } _g_.m.p = 0 _g_.m.mcache = nil p := allp[0] p.m = 0 p.status = _Pidle acquirep(p) if trace.enabled { traceGoStart() } } var runnablePs *p for i := nprocs - 1; i >= 0; i-- { p := allp[i] if _g_.m.p.ptr() == p { continue } p.status = _Pidle if runqempty(p) { pidleput(p) } else { p.m.set(mget()) p.link.set(runnablePs) runnablePs = p } } var int32p *int32 = &gomaxprocs // make compiler check that gomaxprocs is an int32 atomic.Store((*uint32)(unsafe.Pointer(int32p)), uint32(nprocs)) return runnablePs } // Associate p and the current m. func acquirep(_p_ *p) { acquirep1(_p_) // have p; write barriers now allowed _g_ := getg() _g_.m.mcache = _p_.mcache if trace.enabled { traceProcStart() } } // May run during STW, so write barriers are not allowed. //go:nowritebarrier func acquirep1(_p_ *p) { _g_ := getg() if _g_.m.p != 0 || _g_.m.mcache != nil { throw("acquirep: already in go") } if _p_.m != 0 || _p_.status != _Pidle { id := int32(0) if _p_.m != 0 { id = _p_.m.ptr().id } print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n") throw("acquirep: invalid p state") } _g_.m.p.set(_p_) _p_.m.set(_g_.m) _p_.status = _Prunning } // Disassociate p and the current m. func releasep() *p { _g_ := getg() if _g_.m.p == 0 || _g_.m.mcache == nil { throw("releasep: invalid arg") } _p_ := _g_.m.p.ptr() if _p_.m.ptr() != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning { print("releasep: m=", _g_.m, " m->p=", _g_.m.p.ptr(), " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n") throw("releasep: invalid p state") } if trace.enabled { traceProcStop(_g_.m.p.ptr()) } _g_.m.p = 0 _g_.m.mcache = nil _p_.m = 0 _p_.status = _Pidle return _p_ } func incidlelocked(v int32) { lock(&sched.lock) sched.nmidlelocked += v if v > 0 { checkdead() } unlock(&sched.lock) } // Check for deadlock situation. // The check is based on number of running M's, if 0 -> deadlock. func checkdead() { // For -buildmode=c-shared or -buildmode=c-archive it's OK if // there are no running goroutines. The calling program is // assumed to be running. if islibrary || isarchive { return } // If we are dying because of a signal caught on an already idle thread, // freezetheworld will cause all running threads to block. // And runtime will essentially enter into deadlock state, // except that there is a thread that will call exit soon. if panicking > 0 { return } // -1 for sysmon run := sched.mcount - sched.nmidle - sched.nmidlelocked - 1 if run > 0 { return } if run < 0 { print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", sched.mcount, "\n") throw("checkdead: inconsistent counts") } grunning := 0 lock(&allglock) for i := 0; i < len(allgs); i++ { gp := allgs[i] if isSystemGoroutine(gp) { continue } s := readgstatus(gp) switch s &^ _Gscan { case _Gwaiting: grunning++ case _Grunnable, _Grunning, _Gsyscall: unlock(&allglock) print("runtime: checkdead: find g ", gp.goid, " in status ", s, "\n") throw("checkdead: runnable g") } } unlock(&allglock) if grunning == 0 { // possible if main goroutine calls runtime·Goexit() throw("no goroutines (main called runtime.Goexit) - deadlock!") } // Maybe jump time forward for playground. gp := timejump() if gp != nil { casgstatus(gp, _Gwaiting, _Grunnable) globrunqput(gp) _p_ := pidleget() if _p_ == nil { throw("checkdead: no p for timer") } mp := mget() if mp == nil { // There should always be a free M since // nothing is running. throw("checkdead: no m for timer") } mp.nextp.set(_p_) notewakeup(&mp.park) return } getg().m.throwing = -1 // do not dump full stacks throw("all goroutines are asleep - deadlock!") } // forcegcperiod is the maximum time in nanoseconds between garbage // collections. If we go this long without a garbage collection, one // is forced to run. // // This is a variable for testing purposes. It normally doesn't change. var forcegcperiod int64 = 2 * 60 * 1e9 // Always runs without a P, so write barriers are not allowed. // //go:nowritebarrierrec func sysmon() { // If a heap span goes unused for 5 minutes after a garbage collection, // we hand it back to the operating system. scavengelimit := int64(5 * 60 * 1e9) if debug.scavenge > 0 { // Scavenge-a-lot for testing. forcegcperiod = 10 * 1e6 scavengelimit = 20 * 1e6 } lastscavenge := nanotime() nscavenge := 0 lasttrace := int64(0) idle := 0 // how many cycles in succession we had not wokeup somebody delay := uint32(0) for { if idle == 0 { // start with 20us sleep... delay = 20 } else if idle > 50 { // start doubling the sleep after 1ms... delay *= 2 } if delay > 10*1000 { // up to 10ms delay = 10 * 1000 } usleep(delay) if debug.schedtrace <= 0 && (sched.gcwaiting != 0 || atomic.Load(&sched.npidle) == uint32(gomaxprocs)) { // TODO: fast atomic lock(&sched.lock) if atomic.Load(&sched.gcwaiting) != 0 || atomic.Load(&sched.npidle) == uint32(gomaxprocs) { atomic.Store(&sched.sysmonwait, 1) unlock(&sched.lock) // Make wake-up period small enough // for the sampling to be correct. maxsleep := forcegcperiod / 2 if scavengelimit < forcegcperiod { maxsleep = scavengelimit / 2 } notetsleep(&sched.sysmonnote, maxsleep) lock(&sched.lock) atomic.Store(&sched.sysmonwait, 0) noteclear(&sched.sysmonnote) idle = 0 delay = 20 } unlock(&sched.lock) } // poll network if not polled for more than 10ms lastpoll := int64(atomic.Load64(&sched.lastpoll)) now := nanotime() unixnow := unixnanotime() if lastpoll != 0 && lastpoll+10*1000*1000 < now { atomic.Cas64(&sched.lastpoll, uint64(lastpoll), uint64(now)) gp := netpoll(false) // non-blocking - returns list of goroutines if gp != nil { // Need to decrement number of idle locked M's // (pretending that one more is running) before injectglist. // Otherwise it can lead to the following situation: // injectglist grabs all P's but before it starts M's to run the P's, // another M returns from syscall, finishes running its G, // observes that there is no work to do and no other running M's // and reports deadlock. incidlelocked(-1) injectglist(gp) incidlelocked(1) } } // retake P's blocked in syscalls // and preempt long running G's if retake(now) != 0 { idle = 0 } else { idle++ } // check if we need to force a GC lastgc := int64(atomic.Load64(&memstats.last_gc)) if gcphase == _GCoff && lastgc != 0 && unixnow-lastgc > forcegcperiod && atomic.Load(&forcegc.idle) != 0 { lock(&forcegc.lock) forcegc.idle = 0 forcegc.g.schedlink = 0 injectglist(forcegc.g) unlock(&forcegc.lock) } // scavenge heap once in a while if lastscavenge+scavengelimit/2 < now { mheap_.scavenge(int32(nscavenge), uint64(now), uint64(scavengelimit)) lastscavenge = now nscavenge++ } if debug.schedtrace > 0 && lasttrace+int64(debug.schedtrace)*1000000 <= now { lasttrace = now schedtrace(debug.scheddetail > 0) } } } var pdesc [_MaxGomaxprocs]struct { schedtick uint32 schedwhen int64 syscalltick uint32 syscallwhen int64 } // forcePreemptNS is the time slice given to a G before it is // preempted. const forcePreemptNS = 10 * 1000 * 1000 // 10ms func retake(now int64) uint32 { n := 0 for i := int32(0); i < gomaxprocs; i++ { _p_ := allp[i] if _p_ == nil { continue } pd := &pdesc[i] s := _p_.status if s == _Psyscall { // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us). t := int64(_p_.syscalltick) if int64(pd.syscalltick) != t { pd.syscalltick = uint32(t) pd.syscallwhen = now continue } // On the one hand we don't want to retake Ps if there is no other work to do, // but on the other hand we want to retake them eventually // because they can prevent the sysmon thread from deep sleep. if runqempty(_p_) && atomic.Load(&sched.nmspinning)+atomic.Load(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now { continue } // Need to decrement number of idle locked M's // (pretending that one more is running) before the CAS. // Otherwise the M from which we retake can exit the syscall, // increment nmidle and report deadlock. incidlelocked(-1) if atomic.Cas(&_p_.status, s, _Pidle) { if trace.enabled { traceGoSysBlock(_p_) traceProcStop(_p_) } n++ _p_.syscalltick++ handoffp(_p_) } incidlelocked(1) } else if s == _Prunning { // Preempt G if it's running for too long. t := int64(_p_.schedtick) if int64(pd.schedtick) != t { pd.schedtick = uint32(t) pd.schedwhen = now continue } if pd.schedwhen+forcePreemptNS > now { continue } preemptone(_p_) } } return uint32(n) } // Tell all goroutines that they have been preempted and they should stop. // This function is purely best-effort. It can fail to inform a goroutine if a // processor just started running it. // No locks need to be held. // Returns true if preemption request was issued to at least one goroutine. func preemptall() bool { res := false for i := int32(0); i < gomaxprocs; i++ { _p_ := allp[i] if _p_ == nil || _p_.status != _Prunning { continue } if preemptone(_p_) { res = true } } return res } // Tell the goroutine running on processor P to stop. // This function is purely best-effort. It can incorrectly fail to inform the // goroutine. It can send inform the wrong goroutine. Even if it informs the // correct goroutine, that goroutine might ignore the request if it is // simultaneously executing newstack. // No lock needs to be held. // Returns true if preemption request was issued. // The actual preemption will happen at some point in the future // and will be indicated by the gp->status no longer being // Grunning func preemptone(_p_ *p) bool { mp := _p_.m.ptr() if mp == nil || mp == getg().m { return false } gp := mp.curg if gp == nil || gp == mp.g0 { return false } gp.preempt = true // Every call in a go routine checks for stack overflow by // comparing the current stack pointer to gp->stackguard0. // Setting gp->stackguard0 to StackPreempt folds // preemption into the normal stack overflow check. gp.stackguard0 = stackPreempt return true } var starttime int64 func schedtrace(detailed bool) { now := nanotime() if starttime == 0 { starttime = now } lock(&sched.lock) print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", sched.mcount, " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize) if detailed { print(" gcwaiting=", sched.gcwaiting, " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait, "\n") } // We must be careful while reading data from P's, M's and G's. // Even if we hold schedlock, most data can be changed concurrently. // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil. for i := int32(0); i < gomaxprocs; i++ { _p_ := allp[i] if _p_ == nil { continue } mp := _p_.m.ptr() h := atomic.Load(&_p_.runqhead) t := atomic.Load(&_p_.runqtail) if detailed { id := int32(-1) if mp != nil { id = mp.id } print(" P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gfreecnt, "\n") } else { // In non-detailed mode format lengths of per-P run queues as: // [len1 len2 len3 len4] print(" ") if i == 0 { print("[") } print(t - h) if i == gomaxprocs-1 { print("]\n") } } } if !detailed { unlock(&sched.lock) return } for mp := allm; mp != nil; mp = mp.alllink { _p_ := mp.p.ptr() gp := mp.curg lockedg := mp.lockedg id1 := int32(-1) if _p_ != nil { id1 = _p_.id } id2 := int64(-1) if gp != nil { id2 = gp.goid } id3 := int64(-1) if lockedg != nil { id3 = lockedg.goid } print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " preemptoff=", mp.preemptoff, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", getg().m.blocked, " lockedg=", id3, "\n") } lock(&allglock) for gi := 0; gi < len(allgs); gi++ { gp := allgs[gi] mp := gp.m lockedm := gp.lockedm id1 := int32(-1) if mp != nil { id1 = mp.id } id2 := int32(-1) if lockedm != nil { id2 = lockedm.id } print(" G", gp.goid, ": status=", readgstatus(gp), "(", gp.waitreason, ") m=", id1, " lockedm=", id2, "\n") } unlock(&allglock) unlock(&sched.lock) } // Put mp on midle list. // Sched must be locked. // May run during STW, so write barriers are not allowed. //go:nowritebarrier func mput(mp *m) { mp.schedlink = sched.midle sched.midle.set(mp) sched.nmidle++ checkdead() } // Try to get an m from midle list. // Sched must be locked. // May run during STW, so write barriers are not allowed. //go:nowritebarrier func mget() *m { mp := sched.midle.ptr() if mp != nil { sched.midle = mp.schedlink sched.nmidle-- } return mp } // Put gp on the global runnable queue. // Sched must be locked. // May run during STW, so write barriers are not allowed. //go:nowritebarrier func globrunqput(gp *g) { gp.schedlink = 0 if sched.runqtail != 0 { sched.runqtail.ptr().schedlink.set(gp) } else { sched.runqhead.set(gp) } sched.runqtail.set(gp) sched.runqsize++ } // Put gp at the head of the global runnable queue. // Sched must be locked. // May run during STW, so write barriers are not allowed. //go:nowritebarrier func globrunqputhead(gp *g) { gp.schedlink = sched.runqhead sched.runqhead.set(gp) if sched.runqtail == 0 { sched.runqtail.set(gp) } sched.runqsize++ } // Put a batch of runnable goroutines on the global runnable queue. // Sched must be locked. func globrunqputbatch(ghead *g, gtail *g, n int32) { gtail.schedlink = 0 if sched.runqtail != 0 { sched.runqtail.ptr().schedlink.set(ghead) } else { sched.runqhead.set(ghead) } sched.runqtail.set(gtail) sched.runqsize += n } // Try get a batch of G's from the global runnable queue. // Sched must be locked. func globrunqget(_p_ *p, max int32) *g { if sched.runqsize == 0 { return nil } n := sched.runqsize/gomaxprocs + 1 if n > sched.runqsize { n = sched.runqsize } if max > 0 && n > max { n = max } if n > int32(len(_p_.runq))/2 { n = int32(len(_p_.runq)) / 2 } sched.runqsize -= n if sched.runqsize == 0 { sched.runqtail = 0 } gp := sched.runqhead.ptr() sched.runqhead = gp.schedlink n-- for ; n > 0; n-- { gp1 := sched.runqhead.ptr() sched.runqhead = gp1.schedlink runqput(_p_, gp1, false) } return gp } // Put p to on _Pidle list. // Sched must be locked. // May run during STW, so write barriers are not allowed. //go:nowritebarrier func pidleput(_p_ *p) { if !runqempty(_p_) { throw("pidleput: P has non-empty run queue") } _p_.link = sched.pidle sched.pidle.set(_p_) atomic.Xadd(&sched.npidle, 1) // TODO: fast atomic } // Try get a p from _Pidle list. // Sched must be locked. // May run during STW, so write barriers are not allowed. //go:nowritebarrier func pidleget() *p { _p_ := sched.pidle.ptr() if _p_ != nil { sched.pidle = _p_.link atomic.Xadd(&sched.npidle, -1) // TODO: fast atomic } return _p_ } // runqempty returns true if _p_ has no Gs on its local run queue. // Note that this test is generally racy. func runqempty(_p_ *p) bool { return _p_.runqhead == _p_.runqtail && _p_.runnext == 0 } // To shake out latent assumptions about scheduling order, // we introduce some randomness into scheduling decisions // when running with the race detector. // The need for this was made obvious by changing the // (deterministic) scheduling order in Go 1.5 and breaking // many poorly-written tests. // With the randomness here, as long as the tests pass // consistently with -race, they shouldn't have latent scheduling // assumptions. const randomizeScheduler = raceenabled // runqput tries to put g on the local runnable queue. // If next if false, runqput adds g to the tail of the runnable queue. // If next is true, runqput puts g in the _p_.runnext slot. // If the run queue is full, runnext puts g on the global queue. // Executed only by the owner P. func runqput(_p_ *p, gp *g, next bool) { if randomizeScheduler && next && fastrand1()%2 == 0 { next = false } if next { retryNext: oldnext := _p_.runnext if !_p_.runnext.cas(oldnext, guintptr(unsafe.Pointer(gp))) { goto retryNext } if oldnext == 0 { return } // Kick the old runnext out to the regular run queue. gp = oldnext.ptr() } retry: h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with consumers t := _p_.runqtail if t-h < uint32(len(_p_.runq)) { _p_.runq[t%uint32(len(_p_.runq))].set(gp) atomic.Store(&_p_.runqtail, t+1) // store-release, makes the item available for consumption return } if runqputslow(_p_, gp, h, t) { return } // the queue is not full, now the put above must suceed goto retry } // Put g and a batch of work from local runnable queue on global queue. // Executed only by the owner P. func runqputslow(_p_ *p, gp *g, h, t uint32) bool { var batch [len(_p_.runq)/2 + 1]*g // First, grab a batch from local queue. n := t - h n = n / 2 if n != uint32(len(_p_.runq)/2) { throw("runqputslow: queue is not full") } for i := uint32(0); i < n; i++ { batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))].ptr() } if !atomic.Cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume return false } batch[n] = gp if randomizeScheduler { for i := uint32(1); i <= n; i++ { j := fastrand1() % (i + 1) batch[i], batch[j] = batch[j], batch[i] } } // Link the goroutines. for i := uint32(0); i < n; i++ { batch[i].schedlink.set(batch[i+1]) } // Now put the batch on global queue. lock(&sched.lock) globrunqputbatch(batch[0], batch[n], int32(n+1)) unlock(&sched.lock) return true } // Get g from local runnable queue. // If inheritTime is true, gp should inherit the remaining time in the // current time slice. Otherwise, it should start a new time slice. // Executed only by the owner P. func runqget(_p_ *p) (gp *g, inheritTime bool) { // If there's a runnext, it's the next G to run. for { next := _p_.runnext if next == 0 { break } if _p_.runnext.cas(next, 0) { return next.ptr(), true } } for { h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with other consumers t := _p_.runqtail if t == h { return nil, false } gp := _p_.runq[h%uint32(len(_p_.runq))].ptr() if atomic.Cas(&_p_.runqhead, h, h+1) { // cas-release, commits consume return gp, false } } } // Grabs a batch of goroutines from _p_'s runnable queue into batch. // Batch is a ring buffer starting at batchHead. // Returns number of grabbed goroutines. // Can be executed by any P. func runqgrab(_p_ *p, batch *[256]guintptr, batchHead uint32, stealRunNextG bool) uint32 { for { h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with other consumers t := atomic.Load(&_p_.runqtail) // load-acquire, synchronize with the producer n := t - h n = n - n/2 if n == 0 { if stealRunNextG { // Try to steal from _p_.runnext. if next := _p_.runnext; next != 0 { // Sleep to ensure that _p_ isn't about to run the g we // are about to steal. // The important use case here is when the g running on _p_ // ready()s another g and then almost immediately blocks. // Instead of stealing runnext in this window, back off // to give _p_ a chance to schedule runnext. This will avoid // thrashing gs between different Ps. usleep(100) if !_p_.runnext.cas(next, 0) { continue } batch[batchHead%uint32(len(batch))] = next return 1 } } return 0 } if n > uint32(len(_p_.runq)/2) { // read inconsistent h and t continue } for i := uint32(0); i < n; i++ { g := _p_.runq[(h+i)%uint32(len(_p_.runq))] batch[(batchHead+i)%uint32(len(batch))] = g } if atomic.Cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume return n } } } // Steal half of elements from local runnable queue of p2 // and put onto local runnable queue of p. // Returns one of the stolen elements (or nil if failed). func runqsteal(_p_, p2 *p, stealRunNextG bool) *g { t := _p_.runqtail n := runqgrab(p2, &_p_.runq, t, stealRunNextG) if n == 0 { return nil } n-- gp := _p_.runq[(t+n)%uint32(len(_p_.runq))].ptr() if n == 0 { return gp } h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with consumers if t-h+n >= uint32(len(_p_.runq)) { throw("runqsteal: runq overflow") } atomic.Store(&_p_.runqtail, t+n) // store-release, makes the item available for consumption return gp } func testSchedLocalQueue() { _p_ := new(p) gs := make([]g, len(_p_.runq)) for i := 0; i < len(_p_.runq); i++ { if g, _ := runqget(_p_); g != nil { throw("runq is not empty initially") } for j := 0; j < i; j++ { runqput(_p_, &gs[i], false) } for j := 0; j < i; j++ { if g, _ := runqget(_p_); g != &gs[i] { print("bad element at iter ", i, "/", j, "\n") throw("bad element") } } if g, _ := runqget(_p_); g != nil { throw("runq is not empty afterwards") } } } func testSchedLocalQueueSteal() { p1 := new(p) p2 := new(p) gs := make([]g, len(p1.runq)) for i := 0; i < len(p1.runq); i++ { for j := 0; j < i; j++ { gs[j].sig = 0 runqput(p1, &gs[j], false) } gp := runqsteal(p2, p1, true) s := 0 if gp != nil { s++ gp.sig++ } for { gp, _ = runqget(p2) if gp == nil { break } s++ gp.sig++ } for { gp, _ = runqget(p1) if gp == nil { break } gp.sig++ } for j := 0; j < i; j++ { if gs[j].sig != 1 { print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n") throw("bad element") } } if s != i/2 && s != i/2+1 { print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n") throw("bad steal") } } } //go:linkname setMaxThreads runtime/debug.setMaxThreads func setMaxThreads(in int) (out int) { lock(&sched.lock) out = int(sched.maxmcount) sched.maxmcount = int32(in) checkmcount() unlock(&sched.lock) return } func haveexperiment(name string) bool { x := sys.Goexperiment for x != "" { xname := "" i := index(x, ",") if i < 0 { xname, x = x, "" } else { xname, x = x[:i], x[i+1:] } if xname == name { return true } } return false } //go:nosplit func procPin() int { _g_ := getg() mp := _g_.m mp.locks++ return int(mp.p.ptr().id) } //go:nosplit func procUnpin() { _g_ := getg() _g_.m.locks-- } //go:linkname sync_runtime_procPin sync.runtime_procPin //go:nosplit func sync_runtime_procPin() int { return procPin() } //go:linkname sync_runtime_procUnpin sync.runtime_procUnpin //go:nosplit func sync_runtime_procUnpin() { procUnpin() } //go:linkname sync_atomic_runtime_procPin sync/atomic.runtime_procPin //go:nosplit func sync_atomic_runtime_procPin() int { return procPin() } //go:linkname sync_atomic_runtime_procUnpin sync/atomic.runtime_procUnpin //go:nosplit func sync_atomic_runtime_procUnpin() { procUnpin() } // Active spinning for sync.Mutex. //go:linkname sync_runtime_canSpin sync.runtime_canSpin //go:nosplit func sync_runtime_canSpin(i int) bool { // sync.Mutex is cooperative, so we are conservative with spinning. // Spin only few times and only if running on a multicore machine and // GOMAXPROCS>1 and there is at least one other running P and local runq is empty. // As opposed to runtime mutex we don't do passive spinning here, // because there can be work on global runq on on other Ps. if i >= active_spin || ncpu <= 1 || gomaxprocs <= int32(sched.npidle+sched.nmspinning)+1 { return false } if p := getg().m.p.ptr(); !runqempty(p) { return false } return true } //go:linkname sync_runtime_doSpin sync.runtime_doSpin //go:nosplit func sync_runtime_doSpin() { procyield(active_spin_cnt) } ================================================ FILE: examples/go/small.go ================================================ package example type Person struct { name string mom *Person } func NewPerson(name string, mom *Person) Person { return Person{name: name, mom: mom} } func (self *Person) GetName() string { return self.name } func (self *Person) GetMom() *Person { return self.mom } var people = []Person{ Person{name: "Pebbles", mom: "Wilma"}, Person{name: "Wilma", mom: "Pearl"}, } func main() { for p := range people { println(p) } } ================================================ FILE: examples/go/type_switch.go ================================================ package p func f(a interface{}) { switch aa := a.(type) { case *int: print(aa) } } ================================================ FILE: examples/go/value.go ================================================ // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package reflect import ( "math" "runtime" "unsafe" ) const ptrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ideal const const cannotSet = "cannot set value obtained from unexported struct field" // Value is the reflection interface to a Go value. // // Not all methods apply to all kinds of values. Restrictions, // if any, are noted in the documentation for each method. // Use the Kind method to find out the kind of value before // calling kind-specific methods. Calling a method // inappropriate to the kind of type causes a run time panic. // // The zero Value represents no value. // Its IsValid method returns false, its Kind method returns Invalid, // its String method returns "", and all other methods panic. // Most functions and methods never return an invalid value. // If one does, its documentation states the conditions explicitly. // // A Value can be used concurrently by multiple goroutines provided that // the underlying Go value can be used concurrently for the equivalent // direct operations. // // Using == on two Values does not compare the underlying values // they represent, but rather the contents of the Value structs. // To compare two Values, compare the results of the Interface method. type Value struct { // typ holds the type of the value represented by a Value. typ *rtype // Pointer-valued data or, if flagIndir is set, pointer to data. // Valid when either flagIndir is set or typ.pointers() is true. ptr unsafe.Pointer // flag holds metadata about the value. // The lowest bits are flag bits: // - flagStickyRO: obtained via unexported not embedded field, so read-only // - flagEmbedRO: obtained via unexported embedded field, so read-only // - flagIndir: val holds a pointer to the data // - flagAddr: v.CanAddr is true (implies flagIndir) // - flagMethod: v is a method value. // The next five bits give the Kind of the value. // This repeats typ.Kind() except for method values. // The remaining 23+ bits give a method number for method values. // If flag.kind() != Func, code can assume that flagMethod is unset. // If ifaceIndir(typ), code can assume that flagIndir is set. flag // A method value represents a curried method invocation // like r.Read for some receiver r. The typ+val+flag bits describe // the receiver r, but the flag's Kind bits say Func (methods are // functions), and the top bits of the flag give the method number // in r's type's method table. } type flag uintptr const ( flagKindWidth = 5 // there are 27 kinds flagKindMask flag = 1<>flagMethodShift) } else if v.flag&flagIndir != 0 { fn = *(*unsafe.Pointer)(v.ptr) } else { fn = v.ptr } if fn == nil { panic("reflect.Value.Call: call of nil function") } isSlice := op == "CallSlice" n := t.NumIn() if isSlice { if !t.IsVariadic() { panic("reflect: CallSlice of non-variadic function") } if len(in) < n { panic("reflect: CallSlice with too few input arguments") } if len(in) > n { panic("reflect: CallSlice with too many input arguments") } } else { if t.IsVariadic() { n-- } if len(in) < n { panic("reflect: Call with too few input arguments") } if !t.IsVariadic() && len(in) > n { panic("reflect: Call with too many input arguments") } } for _, x := range in { if x.Kind() == Invalid { panic("reflect: " + op + " using zero Value argument") } } for i := 0; i < n; i++ { if xt, targ := in[i].Type(), t.In(i); !xt.AssignableTo(targ) { panic("reflect: " + op + " using " + xt.String() + " as type " + targ.String()) } } if !isSlice && t.IsVariadic() { // prepare slice for remaining values m := len(in) - n slice := MakeSlice(t.In(n), m, m) elem := t.In(n).Elem() for i := 0; i < m; i++ { x := in[n+i] if xt := x.Type(); !xt.AssignableTo(elem) { panic("reflect: cannot use " + xt.String() + " as type " + elem.String() + " in " + op) } slice.Index(i).Set(x) } origIn := in in = make([]Value, n+1) copy(in[:n], origIn) in[n] = slice } nin := len(in) if nin != t.NumIn() { panic("reflect.Value.Call: wrong argument count") } nout := t.NumOut() // Compute frame type. frametype, _, retOffset, _, framePool := funcLayout(t, rcvrtype) // Allocate a chunk of memory for frame. var args unsafe.Pointer if nout == 0 { args = framePool.Get().(unsafe.Pointer) } else { // Can't use pool if the function has return values. // We will leak pointer to args in ret, so its lifetime is not scoped. args = unsafe_New(frametype) } off := uintptr(0) // Copy inputs into args. if rcvrtype != nil { storeRcvr(rcvr, args) off = ptrSize } for i, v := range in { v.mustBeExported() targ := t.In(i).(*rtype) a := uintptr(targ.align) off = (off + a - 1) &^ (a - 1) n := targ.size addr := unsafe.Pointer(uintptr(args) + off) v = v.assignTo("reflect.Value.Call", targ, addr) if v.flag&flagIndir != 0 { typedmemmove(targ, addr, v.ptr) } else { *(*unsafe.Pointer)(addr) = v.ptr } off += n } // Call. call(frametype, fn, args, uint32(frametype.size), uint32(retOffset)) // For testing; see TestCallMethodJump. if callGC { runtime.GC() } var ret []Value if nout == 0 { memclr(args, frametype.size) framePool.Put(args) } else { // Zero the now unused input area of args, // because the Values returned by this function contain pointers to the args object, // and will thus keep the args object alive indefinitely. memclr(args, retOffset) // Copy return values out of args. ret = make([]Value, nout) off = retOffset for i := 0; i < nout; i++ { tv := t.Out(i) a := uintptr(tv.Align()) off = (off + a - 1) &^ (a - 1) fl := flagIndir | flag(tv.Kind()) ret[i] = Value{tv.common(), unsafe.Pointer(uintptr(args) + off), fl} off += tv.Size() } } return ret } // callReflect is the call implementation used by a function // returned by MakeFunc. In many ways it is the opposite of the // method Value.call above. The method above converts a call using Values // into a call of a function with a concrete argument frame, while // callReflect converts a call of a function with a concrete argument // frame into a call using Values. // It is in this file so that it can be next to the call method above. // The remainder of the MakeFunc implementation is in makefunc.go. // // NOTE: This function must be marked as a "wrapper" in the generated code, // so that the linker can make it work correctly for panic and recover. // The gc compilers know to do that for the name "reflect.callReflect". func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) { ftyp := ctxt.typ f := ctxt.fn // Copy argument frame into Values. ptr := frame off := uintptr(0) in := make([]Value, 0, len(ftyp.in)) for _, arg := range ftyp.in { typ := arg off += -off & uintptr(typ.align-1) addr := unsafe.Pointer(uintptr(ptr) + off) v := Value{typ, nil, flag(typ.Kind())} if ifaceIndir(typ) { // value cannot be inlined in interface data. // Must make a copy, because f might keep a reference to it, // and we cannot let f keep a reference to the stack frame // after this function returns, not even a read-only reference. v.ptr = unsafe_New(typ) typedmemmove(typ, v.ptr, addr) v.flag |= flagIndir } else { v.ptr = *(*unsafe.Pointer)(addr) } in = append(in, v) off += typ.size } // Call underlying function. out := f(in) if len(out) != len(ftyp.out) { panic("reflect: wrong return count from function created by MakeFunc") } // Copy results back into argument frame. if len(ftyp.out) > 0 { off += -off & (ptrSize - 1) if runtime.GOARCH == "amd64p32" { off = align(off, 8) } for i, arg := range ftyp.out { typ := arg v := out[i] if v.typ != typ { panic("reflect: function created by MakeFunc using " + funcName(f) + " returned wrong type: have " + out[i].typ.String() + " for " + typ.String()) } if v.flag&flagRO != 0 { panic("reflect: function created by MakeFunc using " + funcName(f) + " returned value obtained from unexported field") } off += -off & uintptr(typ.align-1) addr := unsafe.Pointer(uintptr(ptr) + off) if v.flag&flagIndir != 0 { typedmemmove(typ, addr, v.ptr) } else { *(*unsafe.Pointer)(addr) = v.ptr } off += typ.size } } } // methodReceiver returns information about the receiver // described by v. The Value v may or may not have the // flagMethod bit set, so the kind cached in v.flag should // not be used. // The return value rcvrtype gives the method's actual receiver type. // The return value t gives the method type signature (without the receiver). // The return value fn is a pointer to the method code. func methodReceiver(op string, v Value, methodIndex int) (rcvrtype, t *rtype, fn unsafe.Pointer) { i := methodIndex if v.typ.Kind() == Interface { tt := (*interfaceType)(unsafe.Pointer(v.typ)) if uint(i) >= uint(len(tt.methods)) { panic("reflect: internal error: invalid method index") } m := &tt.methods[i] if m.pkgPath != nil { panic("reflect: " + op + " of unexported method") } iface := (*nonEmptyInterface)(v.ptr) if iface.itab == nil { panic("reflect: " + op + " of method on nil interface value") } rcvrtype = iface.itab.typ fn = unsafe.Pointer(&iface.itab.fun[i]) t = m.typ } else { rcvrtype = v.typ ut := v.typ.uncommon() if ut == nil || uint(i) >= uint(len(ut.methods)) { panic("reflect: internal error: invalid method index") } m := &ut.methods[i] if m.pkgPath != nil { panic("reflect: " + op + " of unexported method") } fn = unsafe.Pointer(&m.ifn) t = m.mtyp } return } // v is a method receiver. Store at p the word which is used to // encode that receiver at the start of the argument list. // Reflect uses the "interface" calling convention for // methods, which always uses one word to record the receiver. func storeRcvr(v Value, p unsafe.Pointer) { t := v.typ if t.Kind() == Interface { // the interface data word becomes the receiver word iface := (*nonEmptyInterface)(v.ptr) *(*unsafe.Pointer)(p) = iface.word } else if v.flag&flagIndir != 0 && !ifaceIndir(t) { *(*unsafe.Pointer)(p) = *(*unsafe.Pointer)(v.ptr) } else { *(*unsafe.Pointer)(p) = v.ptr } } // align returns the result of rounding x up to a multiple of n. // n must be a power of two. func align(x, n uintptr) uintptr { return (x + n - 1) &^ (n - 1) } // callMethod is the call implementation used by a function returned // by makeMethodValue (used by v.Method(i).Interface()). // It is a streamlined version of the usual reflect call: the caller has // already laid out the argument frame for us, so we don't have // to deal with individual Values for each argument. // It is in this file so that it can be next to the two similar functions above. // The remainder of the makeMethodValue implementation is in makefunc.go. // // NOTE: This function must be marked as a "wrapper" in the generated code, // so that the linker can make it work correctly for panic and recover. // The gc compilers know to do that for the name "reflect.callMethod". func callMethod(ctxt *methodValue, frame unsafe.Pointer) { rcvr := ctxt.rcvr rcvrtype, t, fn := methodReceiver("call", rcvr, ctxt.method) frametype, argSize, retOffset, _, framePool := funcLayout(t, rcvrtype) // Make a new frame that is one word bigger so we can store the receiver. args := framePool.Get().(unsafe.Pointer) // Copy in receiver and rest of args. storeRcvr(rcvr, args) typedmemmovepartial(frametype, unsafe.Pointer(uintptr(args)+ptrSize), frame, ptrSize, argSize-ptrSize) // Call. call(frametype, fn, args, uint32(frametype.size), uint32(retOffset)) // Copy return values. On amd64p32, the beginning of return values // is 64-bit aligned, so the caller's frame layout (which doesn't have // a receiver) is different from the layout of the fn call, which has // a receiver. // Ignore any changes to args and just copy return values. callerRetOffset := retOffset - ptrSize if runtime.GOARCH == "amd64p32" { callerRetOffset = align(argSize-ptrSize, 8) } typedmemmovepartial(frametype, unsafe.Pointer(uintptr(frame)+callerRetOffset), unsafe.Pointer(uintptr(args)+retOffset), retOffset, frametype.size-retOffset) memclr(args, frametype.size) framePool.Put(args) } // funcName returns the name of f, for use in error messages. func funcName(f func([]Value) []Value) string { pc := *(*uintptr)(unsafe.Pointer(&f)) rf := runtime.FuncForPC(pc) if rf != nil { return rf.Name() } return "closure" } // Cap returns v's capacity. // It panics if v's Kind is not Array, Chan, or Slice. func (v Value) Cap() int { k := v.kind() switch k { case Array: return v.typ.Len() case Chan: return int(chancap(v.pointer())) case Slice: // Slice is always bigger than a word; assume flagIndir. return (*sliceHeader)(v.ptr).Cap } panic(&ValueError{"reflect.Value.Cap", v.kind()}) } // Close closes the channel v. // It panics if v's Kind is not Chan. func (v Value) Close() { v.mustBe(Chan) v.mustBeExported() chanclose(v.pointer()) } // Complex returns v's underlying value, as a complex128. // It panics if v's Kind is not Complex64 or Complex128 func (v Value) Complex() complex128 { k := v.kind() switch k { case Complex64: return complex128(*(*complex64)(v.ptr)) case Complex128: return *(*complex128)(v.ptr) } panic(&ValueError{"reflect.Value.Complex", v.kind()}) } // Elem returns the value that the interface v contains // or that the pointer v points to. // It panics if v's Kind is not Interface or Ptr. // It returns the zero Value if v is nil. func (v Value) Elem() Value { k := v.kind() switch k { case Interface: var eface interface{} if v.typ.NumMethod() == 0 { eface = *(*interface{})(v.ptr) } else { eface = (interface{})(*(*interface { M() })(v.ptr)) } x := unpackEface(eface) if x.flag != 0 { x.flag |= v.flag & flagRO } return x case Ptr: ptr := v.ptr if v.flag&flagIndir != 0 { ptr = *(*unsafe.Pointer)(ptr) } // The returned value's address is v's value. if ptr == nil { return Value{} } tt := (*ptrType)(unsafe.Pointer(v.typ)) typ := tt.elem fl := v.flag&flagRO | flagIndir | flagAddr fl |= flag(typ.Kind()) return Value{typ, ptr, fl} } panic(&ValueError{"reflect.Value.Elem", v.kind()}) } // Field returns the i'th field of the struct v. // It panics if v's Kind is not Struct or i is out of range. func (v Value) Field(i int) Value { if v.kind() != Struct { panic(&ValueError{"reflect.Value.Field", v.kind()}) } tt := (*structType)(unsafe.Pointer(v.typ)) if uint(i) >= uint(len(tt.fields)) { panic("reflect: Field index out of range") } field := &tt.fields[i] typ := field.typ // Inherit permission bits from v, but clear flagEmbedRO. fl := v.flag&(flagStickyRO|flagIndir|flagAddr) | flag(typ.Kind()) // Using an unexported field forces flagRO. if field.pkgPath != nil { if field.name == nil { fl |= flagEmbedRO } else { fl |= flagStickyRO } } // Either flagIndir is set and v.ptr points at struct, // or flagIndir is not set and v.ptr is the actual struct data. // In the former case, we want v.ptr + offset. // In the latter case, we must have field.offset = 0, // so v.ptr + field.offset is still okay. ptr := unsafe.Pointer(uintptr(v.ptr) + field.offset) return Value{typ, ptr, fl} } // FieldByIndex returns the nested field corresponding to index. // It panics if v's Kind is not struct. func (v Value) FieldByIndex(index []int) Value { if len(index) == 1 { return v.Field(index[0]) } v.mustBe(Struct) for i, x := range index { if i > 0 { if v.Kind() == Ptr && v.typ.Elem().Kind() == Struct { if v.IsNil() { panic("reflect: indirection through nil pointer to embedded struct") } v = v.Elem() } } v = v.Field(x) } return v } // FieldByName returns the struct field with the given name. // It returns the zero Value if no field was found. // It panics if v's Kind is not struct. func (v Value) FieldByName(name string) Value { v.mustBe(Struct) if f, ok := v.typ.FieldByName(name); ok { return v.FieldByIndex(f.Index) } return Value{} } // FieldByNameFunc returns the struct field with a name // that satisfies the match function. // It panics if v's Kind is not struct. // It returns the zero Value if no field was found. func (v Value) FieldByNameFunc(match func(string) bool) Value { if f, ok := v.typ.FieldByNameFunc(match); ok { return v.FieldByIndex(f.Index) } return Value{} } // Float returns v's underlying value, as a float64. // It panics if v's Kind is not Float32 or Float64 func (v Value) Float() float64 { k := v.kind() switch k { case Float32: return float64(*(*float32)(v.ptr)) case Float64: return *(*float64)(v.ptr) } panic(&ValueError{"reflect.Value.Float", v.kind()}) } var uint8Type = TypeOf(uint8(0)).(*rtype) // Index returns v's i'th element. // It panics if v's Kind is not Array, Slice, or String or i is out of range. func (v Value) Index(i int) Value { switch v.kind() { case Array: tt := (*arrayType)(unsafe.Pointer(v.typ)) if uint(i) >= uint(tt.len) { panic("reflect: array index out of range") } typ := tt.elem offset := uintptr(i) * typ.size // Either flagIndir is set and v.ptr points at array, // or flagIndir is not set and v.ptr is the actual array data. // In the former case, we want v.ptr + offset. // In the latter case, we must be doing Index(0), so offset = 0, // so v.ptr + offset is still okay. val := unsafe.Pointer(uintptr(v.ptr) + offset) fl := v.flag&(flagRO|flagIndir|flagAddr) | flag(typ.Kind()) // bits same as overall array return Value{typ, val, fl} case Slice: // Element flag same as Elem of Ptr. // Addressable, indirect, possibly read-only. s := (*sliceHeader)(v.ptr) if uint(i) >= uint(s.Len) { panic("reflect: slice index out of range") } tt := (*sliceType)(unsafe.Pointer(v.typ)) typ := tt.elem val := arrayAt(s.Data, i, typ.size) fl := flagAddr | flagIndir | v.flag&flagRO | flag(typ.Kind()) return Value{typ, val, fl} case String: s := (*stringHeader)(v.ptr) if uint(i) >= uint(s.Len) { panic("reflect: string index out of range") } p := arrayAt(s.Data, i, 1) fl := v.flag&flagRO | flag(Uint8) | flagIndir return Value{uint8Type, p, fl} } panic(&ValueError{"reflect.Value.Index", v.kind()}) } // Int returns v's underlying value, as an int64. // It panics if v's Kind is not Int, Int8, Int16, Int32, or Int64. func (v Value) Int() int64 { k := v.kind() p := v.ptr switch k { case Int: return int64(*(*int)(p)) case Int8: return int64(*(*int8)(p)) case Int16: return int64(*(*int16)(p)) case Int32: return int64(*(*int32)(p)) case Int64: return int64(*(*int64)(p)) } panic(&ValueError{"reflect.Value.Int", v.kind()}) } // CanInterface reports whether Interface can be used without panicking. func (v Value) CanInterface() bool { if v.flag == 0 { panic(&ValueError{"reflect.Value.CanInterface", Invalid}) } return v.flag&flagRO == 0 } // Interface returns v's current value as an interface{}. // It is equivalent to: // var i interface{} = (v's underlying value) // It panics if the Value was obtained by accessing // unexported struct fields. func (v Value) Interface() (i interface{}) { return valueInterface(v, true) } func valueInterface(v Value, safe bool) interface{} { if v.flag == 0 { panic(&ValueError{"reflect.Value.Interface", 0}) } if safe && v.flag&flagRO != 0 { // Do not allow access to unexported values via Interface, // because they might be pointers that should not be // writable or methods or function that should not be callable. panic("reflect.Value.Interface: cannot return value obtained from unexported field or method") } if v.flag&flagMethod != 0 { v = makeMethodValue("Interface", v) } if v.kind() == Interface { // Special case: return the element inside the interface. // Empty interface has one layout, all interfaces with // methods have a second layout. if v.NumMethod() == 0 { return *(*interface{})(v.ptr) } return *(*interface { M() })(v.ptr) } // TODO: pass safe to packEface so we don't need to copy if safe==true? return packEface(v) } // InterfaceData returns the interface v's value as a uintptr pair. // It panics if v's Kind is not Interface. func (v Value) InterfaceData() [2]uintptr { // TODO: deprecate this v.mustBe(Interface) // We treat this as a read operation, so we allow // it even for unexported data, because the caller // has to import "unsafe" to turn it into something // that can be abused. // Interface value is always bigger than a word; assume flagIndir. return *(*[2]uintptr)(v.ptr) } // IsNil reports whether its argument v is nil. The argument must be // a chan, func, interface, map, pointer, or slice value; if it is // not, IsNil panics. Note that IsNil is not always equivalent to a // regular comparison with nil in Go. For example, if v was created // by calling ValueOf with an uninitialized interface variable i, // i==nil will be true but v.IsNil will panic as v will be the zero // Value. func (v Value) IsNil() bool { k := v.kind() switch k { case Chan, Func, Map, Ptr: if v.flag&flagMethod != 0 { return false } ptr := v.ptr if v.flag&flagIndir != 0 { ptr = *(*unsafe.Pointer)(ptr) } return ptr == nil case Interface, Slice: // Both interface and slice are nil if first word is 0. // Both are always bigger than a word; assume flagIndir. return *(*unsafe.Pointer)(v.ptr) == nil } panic(&ValueError{"reflect.Value.IsNil", v.kind()}) } // IsValid reports whether v represents a value. // It returns false if v is the zero Value. // If IsValid returns false, all other methods except String panic. // Most functions and methods never return an invalid value. // If one does, its documentation states the conditions explicitly. func (v Value) IsValid() bool { return v.flag != 0 } // Kind returns v's Kind. // If v is the zero Value (IsValid returns false), Kind returns Invalid. func (v Value) Kind() Kind { return v.kind() } // Len returns v's length. // It panics if v's Kind is not Array, Chan, Map, Slice, or String. func (v Value) Len() int { k := v.kind() switch k { case Array: tt := (*arrayType)(unsafe.Pointer(v.typ)) return int(tt.len) case Chan: return chanlen(v.pointer()) case Map: return maplen(v.pointer()) case Slice: // Slice is bigger than a word; assume flagIndir. return (*sliceHeader)(v.ptr).Len case String: // String is bigger than a word; assume flagIndir. return (*stringHeader)(v.ptr).Len } panic(&ValueError{"reflect.Value.Len", v.kind()}) } // MapIndex returns the value associated with key in the map v. // It panics if v's Kind is not Map. // It returns the zero Value if key is not found in the map or if v represents a nil map. // As in Go, the key's value must be assignable to the map's key type. func (v Value) MapIndex(key Value) Value { v.mustBe(Map) tt := (*mapType)(unsafe.Pointer(v.typ)) // Do not require key to be exported, so that DeepEqual // and other programs can use all the keys returned by // MapKeys as arguments to MapIndex. If either the map // or the key is unexported, though, the result will be // considered unexported. This is consistent with the // behavior for structs, which allow read but not write // of unexported fields. key = key.assignTo("reflect.Value.MapIndex", tt.key, nil) var k unsafe.Pointer if key.flag&flagIndir != 0 { k = key.ptr } else { k = unsafe.Pointer(&key.ptr) } e := mapaccess(v.typ, v.pointer(), k) if e == nil { return Value{} } typ := tt.elem fl := (v.flag | key.flag) & flagRO fl |= flag(typ.Kind()) if ifaceIndir(typ) { // Copy result so future changes to the map // won't change the underlying value. c := unsafe_New(typ) typedmemmove(typ, c, e) return Value{typ, c, fl | flagIndir} } else { return Value{typ, *(*unsafe.Pointer)(e), fl} } } // MapKeys returns a slice containing all the keys present in the map, // in unspecified order. // It panics if v's Kind is not Map. // It returns an empty slice if v represents a nil map. func (v Value) MapKeys() []Value { v.mustBe(Map) tt := (*mapType)(unsafe.Pointer(v.typ)) keyType := tt.key fl := v.flag&flagRO | flag(keyType.Kind()) m := v.pointer() mlen := int(0) if m != nil { mlen = maplen(m) } it := mapiterinit(v.typ, m) a := make([]Value, mlen) var i int for i = 0; i < len(a); i++ { key := mapiterkey(it) if key == nil { // Someone deleted an entry from the map since we // called maplen above. It's a data race, but nothing // we can do about it. break } if ifaceIndir(keyType) { // Copy result so future changes to the map // won't change the underlying value. c := unsafe_New(keyType) typedmemmove(keyType, c, key) a[i] = Value{keyType, c, fl | flagIndir} } else { a[i] = Value{keyType, *(*unsafe.Pointer)(key), fl} } mapiternext(it) } return a[:i] } // Method returns a function value corresponding to v's i'th method. // The arguments to a Call on the returned function should not include // a receiver; the returned function will always use v as the receiver. // Method panics if i is out of range or if v is a nil interface value. func (v Value) Method(i int) Value { if v.typ == nil { panic(&ValueError{"reflect.Value.Method", Invalid}) } if v.flag&flagMethod != 0 || uint(i) >= uint(v.typ.NumMethod()) { panic("reflect: Method index out of range") } if v.typ.Kind() == Interface && v.IsNil() { panic("reflect: Method on nil interface value") } fl := v.flag & (flagStickyRO | flagIndir) // Clear flagEmbedRO fl |= flag(Func) fl |= flag(i)<> (64 - bitSize) return x != trunc } panic(&ValueError{"reflect.Value.OverflowInt", v.kind()}) } // OverflowUint reports whether the uint64 x cannot be represented by v's type. // It panics if v's Kind is not Uint, Uintptr, Uint8, Uint16, Uint32, or Uint64. func (v Value) OverflowUint(x uint64) bool { k := v.kind() switch k { case Uint, Uintptr, Uint8, Uint16, Uint32, Uint64: bitSize := v.typ.size * 8 trunc := (x << (64 - bitSize)) >> (64 - bitSize) return x != trunc } panic(&ValueError{"reflect.Value.OverflowUint", v.kind()}) } // Pointer returns v's value as a uintptr. // It returns uintptr instead of unsafe.Pointer so that // code using reflect cannot obtain unsafe.Pointers // without importing the unsafe package explicitly. // It panics if v's Kind is not Chan, Func, Map, Ptr, Slice, or UnsafePointer. // // If v's Kind is Func, the returned pointer is an underlying // code pointer, but not necessarily enough to identify a // single function uniquely. The only guarantee is that the // result is zero if and only if v is a nil func Value. // // If v's Kind is Slice, the returned pointer is to the first // element of the slice. If the slice is nil the returned value // is 0. If the slice is empty but non-nil the return value is non-zero. func (v Value) Pointer() uintptr { // TODO: deprecate k := v.kind() switch k { case Chan, Map, Ptr, UnsafePointer: return uintptr(v.pointer()) case Func: if v.flag&flagMethod != 0 { // As the doc comment says, the returned pointer is an // underlying code pointer but not necessarily enough to // identify a single function uniquely. All method expressions // created via reflect have the same underlying code pointer, // so their Pointers are equal. The function used here must // match the one used in makeMethodValue. f := methodValueCall return **(**uintptr)(unsafe.Pointer(&f)) } p := v.pointer() // Non-nil func value points at data block. // First word of data block is actual code. if p != nil { p = *(*unsafe.Pointer)(p) } return uintptr(p) case Slice: return (*SliceHeader)(v.ptr).Data } panic(&ValueError{"reflect.Value.Pointer", v.kind()}) } // Recv receives and returns a value from the channel v. // It panics if v's Kind is not Chan. // The receive blocks until a value is ready. // The boolean value ok is true if the value x corresponds to a send // on the channel, false if it is a zero value received because the channel is closed. func (v Value) Recv() (x Value, ok bool) { v.mustBe(Chan) v.mustBeExported() return v.recv(false) } // internal recv, possibly non-blocking (nb). // v is known to be a channel. func (v Value) recv(nb bool) (val Value, ok bool) { tt := (*chanType)(unsafe.Pointer(v.typ)) if ChanDir(tt.dir)&RecvDir == 0 { panic("reflect: recv on send-only channel") } t := tt.elem val = Value{t, nil, flag(t.Kind())} var p unsafe.Pointer if ifaceIndir(t) { p = unsafe_New(t) val.ptr = p val.flag |= flagIndir } else { p = unsafe.Pointer(&val.ptr) } selected, ok := chanrecv(v.typ, v.pointer(), nb, p) if !selected { val = Value{} } return } // Send sends x on the channel v. // It panics if v's kind is not Chan or if x's type is not the same type as v's element type. // As in Go, x's value must be assignable to the channel's element type. func (v Value) Send(x Value) { v.mustBe(Chan) v.mustBeExported() v.send(x, false) } // internal send, possibly non-blocking. // v is known to be a channel. func (v Value) send(x Value, nb bool) (selected bool) { tt := (*chanType)(unsafe.Pointer(v.typ)) if ChanDir(tt.dir)&SendDir == 0 { panic("reflect: send on recv-only channel") } x.mustBeExported() x = x.assignTo("reflect.Value.Send", tt.elem, nil) var p unsafe.Pointer if x.flag&flagIndir != 0 { p = x.ptr } else { p = unsafe.Pointer(&x.ptr) } return chansend(v.typ, v.pointer(), p, nb) } // Set assigns x to the value v. // It panics if CanSet returns false. // As in Go, x's value must be assignable to v's type. func (v Value) Set(x Value) { v.mustBeAssignable() x.mustBeExported() // do not let unexported x leak var target unsafe.Pointer if v.kind() == Interface { target = v.ptr } x = x.assignTo("reflect.Set", v.typ, target) if x.flag&flagIndir != 0 { typedmemmove(v.typ, v.ptr, x.ptr) } else { *(*unsafe.Pointer)(v.ptr) = x.ptr } } // SetBool sets v's underlying value. // It panics if v's Kind is not Bool or if CanSet() is false. func (v Value) SetBool(x bool) { v.mustBeAssignable() v.mustBe(Bool) *(*bool)(v.ptr) = x } // SetBytes sets v's underlying value. // It panics if v's underlying value is not a slice of bytes. func (v Value) SetBytes(x []byte) { v.mustBeAssignable() v.mustBe(Slice) if v.typ.Elem().Kind() != Uint8 { panic("reflect.Value.SetBytes of non-byte slice") } *(*[]byte)(v.ptr) = x } // setRunes sets v's underlying value. // It panics if v's underlying value is not a slice of runes (int32s). func (v Value) setRunes(x []rune) { v.mustBeAssignable() v.mustBe(Slice) if v.typ.Elem().Kind() != Int32 { panic("reflect.Value.setRunes of non-rune slice") } *(*[]rune)(v.ptr) = x } // SetComplex sets v's underlying value to x. // It panics if v's Kind is not Complex64 or Complex128, or if CanSet() is false. func (v Value) SetComplex(x complex128) { v.mustBeAssignable() switch k := v.kind(); k { default: panic(&ValueError{"reflect.Value.SetComplex", v.kind()}) case Complex64: *(*complex64)(v.ptr) = complex64(x) case Complex128: *(*complex128)(v.ptr) = x } } // SetFloat sets v's underlying value to x. // It panics if v's Kind is not Float32 or Float64, or if CanSet() is false. func (v Value) SetFloat(x float64) { v.mustBeAssignable() switch k := v.kind(); k { default: panic(&ValueError{"reflect.Value.SetFloat", v.kind()}) case Float32: *(*float32)(v.ptr) = float32(x) case Float64: *(*float64)(v.ptr) = x } } // SetInt sets v's underlying value to x. // It panics if v's Kind is not Int, Int8, Int16, Int32, or Int64, or if CanSet() is false. func (v Value) SetInt(x int64) { v.mustBeAssignable() switch k := v.kind(); k { default: panic(&ValueError{"reflect.Value.SetInt", v.kind()}) case Int: *(*int)(v.ptr) = int(x) case Int8: *(*int8)(v.ptr) = int8(x) case Int16: *(*int16)(v.ptr) = int16(x) case Int32: *(*int32)(v.ptr) = int32(x) case Int64: *(*int64)(v.ptr) = x } } // SetLen sets v's length to n. // It panics if v's Kind is not Slice or if n is negative or // greater than the capacity of the slice. func (v Value) SetLen(n int) { v.mustBeAssignable() v.mustBe(Slice) s := (*sliceHeader)(v.ptr) if uint(n) > uint(s.Cap) { panic("reflect: slice length out of range in SetLen") } s.Len = n } // SetCap sets v's capacity to n. // It panics if v's Kind is not Slice or if n is smaller than the length or // greater than the capacity of the slice. func (v Value) SetCap(n int) { v.mustBeAssignable() v.mustBe(Slice) s := (*sliceHeader)(v.ptr) if n < int(s.Len) || n > int(s.Cap) { panic("reflect: slice capacity out of range in SetCap") } s.Cap = n } // SetMapIndex sets the value associated with key in the map v to val. // It panics if v's Kind is not Map. // If val is the zero Value, SetMapIndex deletes the key from the map. // Otherwise if v holds a nil map, SetMapIndex will panic. // As in Go, key's value must be assignable to the map's key type, // and val's value must be assignable to the map's value type. func (v Value) SetMapIndex(key, val Value) { v.mustBe(Map) v.mustBeExported() key.mustBeExported() tt := (*mapType)(unsafe.Pointer(v.typ)) key = key.assignTo("reflect.Value.SetMapIndex", tt.key, nil) var k unsafe.Pointer if key.flag&flagIndir != 0 { k = key.ptr } else { k = unsafe.Pointer(&key.ptr) } if val.typ == nil { mapdelete(v.typ, v.pointer(), k) return } val.mustBeExported() val = val.assignTo("reflect.Value.SetMapIndex", tt.elem, nil) var e unsafe.Pointer if val.flag&flagIndir != 0 { e = val.ptr } else { e = unsafe.Pointer(&val.ptr) } mapassign(v.typ, v.pointer(), k, e) } // SetUint sets v's underlying value to x. // It panics if v's Kind is not Uint, Uintptr, Uint8, Uint16, Uint32, or Uint64, or if CanSet() is false. func (v Value) SetUint(x uint64) { v.mustBeAssignable() switch k := v.kind(); k { default: panic(&ValueError{"reflect.Value.SetUint", v.kind()}) case Uint: *(*uint)(v.ptr) = uint(x) case Uint8: *(*uint8)(v.ptr) = uint8(x) case Uint16: *(*uint16)(v.ptr) = uint16(x) case Uint32: *(*uint32)(v.ptr) = uint32(x) case Uint64: *(*uint64)(v.ptr) = x case Uintptr: *(*uintptr)(v.ptr) = uintptr(x) } } // SetPointer sets the unsafe.Pointer value v to x. // It panics if v's Kind is not UnsafePointer. func (v Value) SetPointer(x unsafe.Pointer) { v.mustBeAssignable() v.mustBe(UnsafePointer) *(*unsafe.Pointer)(v.ptr) = x } // SetString sets v's underlying value to x. // It panics if v's Kind is not String or if CanSet() is false. func (v Value) SetString(x string) { v.mustBeAssignable() v.mustBe(String) *(*string)(v.ptr) = x } // Slice returns v[i:j]. // It panics if v's Kind is not Array, Slice or String, or if v is an unaddressable array, // or if the indexes are out of bounds. func (v Value) Slice(i, j int) Value { var ( cap int typ *sliceType base unsafe.Pointer ) switch kind := v.kind(); kind { default: panic(&ValueError{"reflect.Value.Slice", v.kind()}) case Array: if v.flag&flagAddr == 0 { panic("reflect.Value.Slice: slice of unaddressable array") } tt := (*arrayType)(unsafe.Pointer(v.typ)) cap = int(tt.len) typ = (*sliceType)(unsafe.Pointer(tt.slice)) base = v.ptr case Slice: typ = (*sliceType)(unsafe.Pointer(v.typ)) s := (*sliceHeader)(v.ptr) base = unsafe.Pointer(s.Data) cap = s.Cap case String: s := (*stringHeader)(v.ptr) if i < 0 || j < i || j > s.Len { panic("reflect.Value.Slice: string slice index out of bounds") } t := stringHeader{arrayAt(s.Data, i, 1), j - i} return Value{v.typ, unsafe.Pointer(&t), v.flag} } if i < 0 || j < i || j > cap { panic("reflect.Value.Slice: slice index out of bounds") } // Declare slice so that gc can see the base pointer in it. var x []unsafe.Pointer // Reinterpret as *sliceHeader to edit. s := (*sliceHeader)(unsafe.Pointer(&x)) s.Len = j - i s.Cap = cap - i if cap-i > 0 { s.Data = arrayAt(base, i, typ.elem.Size()) } else { // do not advance pointer, to avoid pointing beyond end of slice s.Data = base } fl := v.flag&flagRO | flagIndir | flag(Slice) return Value{typ.common(), unsafe.Pointer(&x), fl} } // Slice3 is the 3-index form of the slice operation: it returns v[i:j:k]. // It panics if v's Kind is not Array or Slice, or if v is an unaddressable array, // or if the indexes are out of bounds. func (v Value) Slice3(i, j, k int) Value { var ( cap int typ *sliceType base unsafe.Pointer ) switch kind := v.kind(); kind { default: panic(&ValueError{"reflect.Value.Slice3", v.kind()}) case Array: if v.flag&flagAddr == 0 { panic("reflect.Value.Slice3: slice of unaddressable array") } tt := (*arrayType)(unsafe.Pointer(v.typ)) cap = int(tt.len) typ = (*sliceType)(unsafe.Pointer(tt.slice)) base = v.ptr case Slice: typ = (*sliceType)(unsafe.Pointer(v.typ)) s := (*sliceHeader)(v.ptr) base = s.Data cap = s.Cap } if i < 0 || j < i || k < j || k > cap { panic("reflect.Value.Slice3: slice index out of bounds") } // Declare slice so that the garbage collector // can see the base pointer in it. var x []unsafe.Pointer // Reinterpret as *sliceHeader to edit. s := (*sliceHeader)(unsafe.Pointer(&x)) s.Len = j - i s.Cap = k - i if k-i > 0 { s.Data = arrayAt(base, i, typ.elem.Size()) } else { // do not advance pointer, to avoid pointing beyond end of slice s.Data = base } fl := v.flag&flagRO | flagIndir | flag(Slice) return Value{typ.common(), unsafe.Pointer(&x), fl} } // String returns the string v's underlying value, as a string. // String is a special case because of Go's String method convention. // Unlike the other getters, it does not panic if v's Kind is not String. // Instead, it returns a string of the form "" where T is v's type. // The fmt package treats Values specially. It does not call their String // method implicitly but instead prints the concrete values they hold. func (v Value) String() string { switch k := v.kind(); k { case Invalid: return "" case String: return *(*string)(v.ptr) } // If you call String on a reflect.Value of other type, it's better to // print something than to panic. Useful in debugging. return "<" + v.Type().String() + " Value>" } // TryRecv attempts to receive a value from the channel v but will not block. // It panics if v's Kind is not Chan. // If the receive delivers a value, x is the transferred value and ok is true. // If the receive cannot finish without blocking, x is the zero Value and ok is false. // If the channel is closed, x is the zero value for the channel's element type and ok is false. func (v Value) TryRecv() (x Value, ok bool) { v.mustBe(Chan) v.mustBeExported() return v.recv(true) } // TrySend attempts to send x on the channel v but will not block. // It panics if v's Kind is not Chan. // It reports whether the value was sent. // As in Go, x's value must be assignable to the channel's element type. func (v Value) TrySend(x Value) bool { v.mustBe(Chan) v.mustBeExported() return v.send(x, true) } // Type returns v's type. func (v Value) Type() Type { f := v.flag if f == 0 { panic(&ValueError{"reflect.Value.Type", Invalid}) } if f&flagMethod == 0 { // Easy case return v.typ } // Method value. // v.typ describes the receiver, not the method type. i := int(v.flag) >> flagMethodShift if v.typ.Kind() == Interface { // Method on interface. tt := (*interfaceType)(unsafe.Pointer(v.typ)) if uint(i) >= uint(len(tt.methods)) { panic("reflect: internal error: invalid method index") } m := &tt.methods[i] return m.typ } // Method on concrete type. ut := v.typ.uncommon() if ut == nil || uint(i) >= uint(len(ut.methods)) { panic("reflect: internal error: invalid method index") } m := &ut.methods[i] return m.mtyp } // Uint returns v's underlying value, as a uint64. // It panics if v's Kind is not Uint, Uintptr, Uint8, Uint16, Uint32, or Uint64. func (v Value) Uint() uint64 { k := v.kind() p := v.ptr switch k { case Uint: return uint64(*(*uint)(p)) case Uint8: return uint64(*(*uint8)(p)) case Uint16: return uint64(*(*uint16)(p)) case Uint32: return uint64(*(*uint32)(p)) case Uint64: return uint64(*(*uint64)(p)) case Uintptr: return uint64(*(*uintptr)(p)) } panic(&ValueError{"reflect.Value.Uint", v.kind()}) } // UnsafeAddr returns a pointer to v's data. // It is for advanced clients that also import the "unsafe" package. // It panics if v is not addressable. func (v Value) UnsafeAddr() uintptr { // TODO: deprecate if v.typ == nil { panic(&ValueError{"reflect.Value.UnsafeAddr", Invalid}) } if v.flag&flagAddr == 0 { panic("reflect.Value.UnsafeAddr of unaddressable value") } return uintptr(v.ptr) } // StringHeader is the runtime representation of a string. // It cannot be used safely or portably and its representation may // change in a later release. // Moreover, the Data field is not sufficient to guarantee the data // it references will not be garbage collected, so programs must keep // a separate, correctly typed pointer to the underlying data. type StringHeader struct { Data uintptr Len int } // stringHeader is a safe version of StringHeader used within this package. type stringHeader struct { Data unsafe.Pointer Len int } // SliceHeader is the runtime representation of a slice. // It cannot be used safely or portably and its representation may // change in a later release. // Moreover, the Data field is not sufficient to guarantee the data // it references will not be garbage collected, so programs must keep // a separate, correctly typed pointer to the underlying data. type SliceHeader struct { Data uintptr Len int Cap int } // sliceHeader is a safe version of SliceHeader used within this package. type sliceHeader struct { Data unsafe.Pointer Len int Cap int } func typesMustMatch(what string, t1, t2 Type) { if t1 != t2 { panic(what + ": " + t1.String() + " != " + t2.String()) } } // arrayAt returns the i-th element of p, a C-array whose elements are // eltSize wide (in bytes). func arrayAt(p unsafe.Pointer, i int, eltSize uintptr) unsafe.Pointer { return unsafe.Pointer(uintptr(p) + uintptr(i)*eltSize) } // grow grows the slice s so that it can hold extra more values, allocating // more capacity if needed. It also returns the old and new slice lengths. func grow(s Value, extra int) (Value, int, int) { i0 := s.Len() i1 := i0 + extra if i1 < i0 { panic("reflect.Append: slice overflow") } m := s.Cap() if i1 <= m { return s.Slice(0, i1), i0, i1 } if m == 0 { m = extra } else { for m < i1 { if i0 < 1024 { m += m } else { m += m / 4 } } } t := MakeSlice(s.Type(), i1, m) Copy(t, s) return t, i0, i1 } // Append appends the values x to a slice s and returns the resulting slice. // As in Go, each x's value must be assignable to the slice's element type. func Append(s Value, x ...Value) Value { s.mustBe(Slice) s, i0, i1 := grow(s, len(x)) for i, j := i0, 0; i < i1; i, j = i+1, j+1 { s.Index(i).Set(x[j]) } return s } // AppendSlice appends a slice t to a slice s and returns the resulting slice. // The slices s and t must have the same element type. func AppendSlice(s, t Value) Value { s.mustBe(Slice) t.mustBe(Slice) typesMustMatch("reflect.AppendSlice", s.Type().Elem(), t.Type().Elem()) s, i0, i1 := grow(s, t.Len()) Copy(s.Slice(i0, i1), t) return s } // Copy copies the contents of src into dst until either // dst has been filled or src has been exhausted. // It returns the number of elements copied. // Dst and src each must have kind Slice or Array, and // dst and src must have the same element type. func Copy(dst, src Value) int { dk := dst.kind() if dk != Array && dk != Slice { panic(&ValueError{"reflect.Copy", dk}) } if dk == Array { dst.mustBeAssignable() } dst.mustBeExported() sk := src.kind() if sk != Array && sk != Slice { panic(&ValueError{"reflect.Copy", sk}) } src.mustBeExported() de := dst.typ.Elem() se := src.typ.Elem() typesMustMatch("reflect.Copy", de, se) var ds, ss sliceHeader if dk == Array { ds.Data = dst.ptr ds.Len = dst.Len() ds.Cap = ds.Len } else { ds = *(*sliceHeader)(dst.ptr) } if sk == Array { ss.Data = src.ptr ss.Len = src.Len() ss.Cap = ss.Len } else { ss = *(*sliceHeader)(src.ptr) } return typedslicecopy(de.common(), ds, ss) } // A runtimeSelect is a single case passed to rselect. // This must match ../runtime/select.go:/runtimeSelect type runtimeSelect struct { dir uintptr // 0, SendDir, or RecvDir typ *rtype // channel type ch unsafe.Pointer // channel val unsafe.Pointer // ptr to data (SendDir) or ptr to receive buffer (RecvDir) } // rselect runs a select. It returns the index of the chosen case. // If the case was a receive, val is filled in with the received value. // The conventional OK bool indicates whether the receive corresponds // to a sent value. //go:noescape func rselect([]runtimeSelect) (chosen int, recvOK bool) // A SelectDir describes the communication direction of a select case. type SelectDir int // NOTE: These values must match ../runtime/select.go:/selectDir. const ( _ SelectDir = iota SelectSend // case Chan <- Send SelectRecv // case <-Chan: SelectDefault // default ) // A SelectCase describes a single case in a select operation. // The kind of case depends on Dir, the communication direction. // // If Dir is SelectDefault, the case represents a default case. // Chan and Send must be zero Values. // // If Dir is SelectSend, the case represents a send operation. // Normally Chan's underlying value must be a channel, and Send's underlying value must be // assignable to the channel's element type. As a special case, if Chan is a zero Value, // then the case is ignored, and the field Send will also be ignored and may be either zero // or non-zero. // // If Dir is SelectRecv, the case represents a receive operation. // Normally Chan's underlying value must be a channel and Send must be a zero Value. // If Chan is a zero Value, then the case is ignored, but Send must still be a zero Value. // When a receive operation is selected, the received Value is returned by Select. // type SelectCase struct { Dir SelectDir // direction of case Chan Value // channel to use (for send or receive) Send Value // value to send (for send) } // Select executes a select operation described by the list of cases. // Like the Go select statement, it blocks until at least one of the cases // can proceed, makes a uniform pseudo-random choice, // and then executes that case. It returns the index of the chosen case // and, if that case was a receive operation, the value received and a // boolean indicating whether the value corresponds to a send on the channel // (as opposed to a zero value received because the channel is closed). func Select(cases []SelectCase) (chosen int, recv Value, recvOK bool) { // NOTE: Do not trust that caller is not modifying cases data underfoot. // The range is safe because the caller cannot modify our copy of the len // and each iteration makes its own copy of the value c. runcases := make([]runtimeSelect, len(cases)) haveDefault := false for i, c := range cases { rc := &runcases[i] rc.dir = uintptr(c.Dir) switch c.Dir { default: panic("reflect.Select: invalid Dir") case SelectDefault: // default if haveDefault { panic("reflect.Select: multiple default cases") } haveDefault = true if c.Chan.IsValid() { panic("reflect.Select: default case has Chan value") } if c.Send.IsValid() { panic("reflect.Select: default case has Send value") } case SelectSend: ch := c.Chan if !ch.IsValid() { break } ch.mustBe(Chan) ch.mustBeExported() tt := (*chanType)(unsafe.Pointer(ch.typ)) if ChanDir(tt.dir)&SendDir == 0 { panic("reflect.Select: SendDir case using recv-only channel") } rc.ch = ch.pointer() rc.typ = &tt.rtype v := c.Send if !v.IsValid() { panic("reflect.Select: SendDir case missing Send value") } v.mustBeExported() v = v.assignTo("reflect.Select", tt.elem, nil) if v.flag&flagIndir != 0 { rc.val = v.ptr } else { rc.val = unsafe.Pointer(&v.ptr) } case SelectRecv: if c.Send.IsValid() { panic("reflect.Select: RecvDir case has Send value") } ch := c.Chan if !ch.IsValid() { break } ch.mustBe(Chan) ch.mustBeExported() tt := (*chanType)(unsafe.Pointer(ch.typ)) if ChanDir(tt.dir)&RecvDir == 0 { panic("reflect.Select: RecvDir case using send-only channel") } rc.ch = ch.pointer() rc.typ = &tt.rtype rc.val = unsafe_New(tt.elem) } } chosen, recvOK = rselect(runcases) if runcases[chosen].dir == uintptr(SelectRecv) { tt := (*chanType)(unsafe.Pointer(runcases[chosen].typ)) t := tt.elem p := runcases[chosen].val fl := flag(t.Kind()) if ifaceIndir(t) { recv = Value{t, p, fl | flagIndir} } else { recv = Value{t, *(*unsafe.Pointer)(p), fl} } } return chosen, recv, recvOK } /* * constructors */ // implemented in package runtime func unsafe_New(*rtype) unsafe.Pointer func unsafe_NewArray(*rtype, int) unsafe.Pointer // MakeSlice creates a new zero-initialized slice value // for the specified slice type, length, and capacity. func MakeSlice(typ Type, len, cap int) Value { if typ.Kind() != Slice { panic("reflect.MakeSlice of non-slice type") } if len < 0 { panic("reflect.MakeSlice: negative len") } if cap < 0 { panic("reflect.MakeSlice: negative cap") } if len > cap { panic("reflect.MakeSlice: len > cap") } s := sliceHeader{unsafe_NewArray(typ.Elem().(*rtype), cap), len, cap} return Value{typ.common(), unsafe.Pointer(&s), flagIndir | flag(Slice)} } // MakeChan creates a new channel with the specified type and buffer size. func MakeChan(typ Type, buffer int) Value { if typ.Kind() != Chan { panic("reflect.MakeChan of non-chan type") } if buffer < 0 { panic("reflect.MakeChan: negative buffer size") } if typ.ChanDir() != BothDir { panic("reflect.MakeChan: unidirectional channel type") } ch := makechan(typ.(*rtype), uint64(buffer)) return Value{typ.common(), ch, flag(Chan)} } // MakeMap creates a new map of the specified type. func MakeMap(typ Type) Value { if typ.Kind() != Map { panic("reflect.MakeMap of non-map type") } m := makemap(typ.(*rtype)) return Value{typ.common(), m, flag(Map)} } // Indirect returns the value that v points to. // If v is a nil pointer, Indirect returns a zero Value. // If v is not a pointer, Indirect returns v. func Indirect(v Value) Value { if v.Kind() != Ptr { return v } return v.Elem() } // ValueOf returns a new Value initialized to the concrete value // stored in the interface i. ValueOf(nil) returns the zero Value. func ValueOf(i interface{}) Value { if i == nil { return Value{} } // TODO: Maybe allow contents of a Value to live on the stack. // For now we make the contents always escape to the heap. It // makes life easier in a few places (see chanrecv/mapassign // comment below). escapes(i) return unpackEface(i) } // Zero returns a Value representing the zero value for the specified type. // The result is different from the zero value of the Value struct, // which represents no value at all. // For example, Zero(TypeOf(42)) returns a Value with Kind Int and value 0. // The returned value is neither addressable nor settable. func Zero(typ Type) Value { if typ == nil { panic("reflect: Zero(nil)") } t := typ.common() fl := flag(t.Kind()) if ifaceIndir(t) { return Value{t, unsafe_New(typ.(*rtype)), fl | flagIndir} } return Value{t, nil, fl} } // New returns a Value representing a pointer to a new zero value // for the specified type. That is, the returned Value's Type is PtrTo(typ). func New(typ Type) Value { if typ == nil { panic("reflect: New(nil)") } ptr := unsafe_New(typ.(*rtype)) fl := flag(Ptr) return Value{typ.common().ptrTo(), ptr, fl} } // NewAt returns a Value representing a pointer to a value of the // specified type, using p as that pointer. func NewAt(typ Type, p unsafe.Pointer) Value { fl := flag(Ptr) return Value{typ.common().ptrTo(), p, fl} } // assignTo returns a value v that can be assigned directly to typ. // It panics if v is not assignable to typ. // For a conversion to an interface type, target is a suggested scratch space to use. func (v Value) assignTo(context string, dst *rtype, target unsafe.Pointer) Value { if v.flag&flagMethod != 0 { v = makeMethodValue(context, v) } switch { case directlyAssignable(dst, v.typ): // Overwrite type so that they match. // Same memory layout, so no harm done. v.typ = dst fl := v.flag & (flagRO | flagAddr | flagIndir) fl |= flag(dst.Kind()) return Value{dst, v.ptr, fl} case implements(dst, v.typ): if target == nil { target = unsafe_New(dst) } x := valueInterface(v, false) if dst.NumMethod() == 0 { *(*interface{})(target) = x } else { ifaceE2I(dst, x, target) } return Value{dst, target, flagIndir | flag(Interface)} } // Failed. panic(context + ": value of type " + v.typ.String() + " is not assignable to type " + dst.String()) } // Convert returns the value v converted to type t. // If the usual Go conversion rules do not allow conversion // of the value v to type t, Convert panics. func (v Value) Convert(t Type) Value { if v.flag&flagMethod != 0 { v = makeMethodValue("Convert", v) } op := convertOp(t.common(), v.typ) if op == nil { panic("reflect.Value.Convert: value of type " + v.typ.String() + " cannot be converted to type " + t.String()) } return op(v, t) } // convertOp returns the function to convert a value of type src // to a value of type dst. If the conversion is illegal, convertOp returns nil. func convertOp(dst, src *rtype) func(Value, Type) Value { switch src.Kind() { case Int, Int8, Int16, Int32, Int64: switch dst.Kind() { case Int, Int8, Int16, Int32, Int64, Uint, Uint8, Uint16, Uint32, Uint64, Uintptr: return cvtInt case Float32, Float64: return cvtIntFloat case String: return cvtIntString } case Uint, Uint8, Uint16, Uint32, Uint64, Uintptr: switch dst.Kind() { case Int, Int8, Int16, Int32, Int64, Uint, Uint8, Uint16, Uint32, Uint64, Uintptr: return cvtUint case Float32, Float64: return cvtUintFloat case String: return cvtUintString } case Float32, Float64: switch dst.Kind() { case Int, Int8, Int16, Int32, Int64: return cvtFloatInt case Uint, Uint8, Uint16, Uint32, Uint64, Uintptr: return cvtFloatUint case Float32, Float64: return cvtFloat } case Complex64, Complex128: switch dst.Kind() { case Complex64, Complex128: return cvtComplex } case String: if dst.Kind() == Slice && dst.Elem().PkgPath() == "" { switch dst.Elem().Kind() { case Uint8: return cvtStringBytes case Int32: return cvtStringRunes } } case Slice: if dst.Kind() == String && src.Elem().PkgPath() == "" { switch src.Elem().Kind() { case Uint8: return cvtBytesString case Int32: return cvtRunesString } } } // dst and src have same underlying type. if haveIdenticalUnderlyingType(dst, src) { return cvtDirect } // dst and src are unnamed pointer types with same underlying base type. if dst.Kind() == Ptr && dst.Name() == "" && src.Kind() == Ptr && src.Name() == "" && haveIdenticalUnderlyingType(dst.Elem().common(), src.Elem().common()) { return cvtDirect } if implements(dst, src) { if src.Kind() == Interface { return cvtI2I } return cvtT2I } return nil } // makeInt returns a Value of type t equal to bits (possibly truncated), // where t is a signed or unsigned int type. func makeInt(f flag, bits uint64, t Type) Value { typ := t.common() ptr := unsafe_New(typ) switch typ.size { case 1: *(*uint8)(unsafe.Pointer(ptr)) = uint8(bits) case 2: *(*uint16)(unsafe.Pointer(ptr)) = uint16(bits) case 4: *(*uint32)(unsafe.Pointer(ptr)) = uint32(bits) case 8: *(*uint64)(unsafe.Pointer(ptr)) = bits } return Value{typ, ptr, f | flagIndir | flag(typ.Kind())} } // makeFloat returns a Value of type t equal to v (possibly truncated to float32), // where t is a float32 or float64 type. func makeFloat(f flag, v float64, t Type) Value { typ := t.common() ptr := unsafe_New(typ) switch typ.size { case 4: *(*float32)(unsafe.Pointer(ptr)) = float32(v) case 8: *(*float64)(unsafe.Pointer(ptr)) = v } return Value{typ, ptr, f | flagIndir | flag(typ.Kind())} } // makeComplex returns a Value of type t equal to v (possibly truncated to complex64), // where t is a complex64 or complex128 type. func makeComplex(f flag, v complex128, t Type) Value { typ := t.common() ptr := unsafe_New(typ) switch typ.size { case 8: *(*complex64)(unsafe.Pointer(ptr)) = complex64(v) case 16: *(*complex128)(unsafe.Pointer(ptr)) = v } return Value{typ, ptr, f | flagIndir | flag(typ.Kind())} } func makeString(f flag, v string, t Type) Value { ret := New(t).Elem() ret.SetString(v) ret.flag = ret.flag&^flagAddr | f return ret } func makeBytes(f flag, v []byte, t Type) Value { ret := New(t).Elem() ret.SetBytes(v) ret.flag = ret.flag&^flagAddr | f return ret } func makeRunes(f flag, v []rune, t Type) Value { ret := New(t).Elem() ret.setRunes(v) ret.flag = ret.flag&^flagAddr | f return ret } // These conversion functions are returned by convertOp // for classes of conversions. For example, the first function, cvtInt, // takes any value v of signed int type and returns the value converted // to type t, where t is any signed or unsigned int type. // convertOp: intXX -> [u]intXX func cvtInt(v Value, t Type) Value { return makeInt(v.flag&flagRO, uint64(v.Int()), t) } // convertOp: uintXX -> [u]intXX func cvtUint(v Value, t Type) Value { return makeInt(v.flag&flagRO, v.Uint(), t) } // convertOp: floatXX -> intXX func cvtFloatInt(v Value, t Type) Value { return makeInt(v.flag&flagRO, uint64(int64(v.Float())), t) } // convertOp: floatXX -> uintXX func cvtFloatUint(v Value, t Type) Value { return makeInt(v.flag&flagRO, uint64(v.Float()), t) } // convertOp: intXX -> floatXX func cvtIntFloat(v Value, t Type) Value { return makeFloat(v.flag&flagRO, float64(v.Int()), t) } // convertOp: uintXX -> floatXX func cvtUintFloat(v Value, t Type) Value { return makeFloat(v.flag&flagRO, float64(v.Uint()), t) } // convertOp: floatXX -> floatXX func cvtFloat(v Value, t Type) Value { return makeFloat(v.flag&flagRO, v.Float(), t) } // convertOp: complexXX -> complexXX func cvtComplex(v Value, t Type) Value { return makeComplex(v.flag&flagRO, v.Complex(), t) } // convertOp: intXX -> string func cvtIntString(v Value, t Type) Value { return makeString(v.flag&flagRO, string(v.Int()), t) } // convertOp: uintXX -> string func cvtUintString(v Value, t Type) Value { return makeString(v.flag&flagRO, string(v.Uint()), t) } // convertOp: []byte -> string func cvtBytesString(v Value, t Type) Value { return makeString(v.flag&flagRO, string(v.Bytes()), t) } // convertOp: string -> []byte func cvtStringBytes(v Value, t Type) Value { return makeBytes(v.flag&flagRO, []byte(v.String()), t) } // convertOp: []rune -> string func cvtRunesString(v Value, t Type) Value { return makeString(v.flag&flagRO, string(v.runes()), t) } // convertOp: string -> []rune func cvtStringRunes(v Value, t Type) Value { return makeRunes(v.flag&flagRO, []rune(v.String()), t) } // convertOp: direct copy func cvtDirect(v Value, typ Type) Value { f := v.flag t := typ.common() ptr := v.ptr if f&flagAddr != 0 { // indirect, mutable word - make a copy c := unsafe_New(t) typedmemmove(t, c, ptr) ptr = c f &^= flagAddr } return Value{t, ptr, v.flag&flagRO | f} // v.flag&flagRO|f == f? } // convertOp: concrete -> interface func cvtT2I(v Value, typ Type) Value { target := unsafe_New(typ.common()) x := valueInterface(v, false) if typ.NumMethod() == 0 { *(*interface{})(target) = x } else { ifaceE2I(typ.(*rtype), x, target) } return Value{typ.common(), target, v.flag&flagRO | flagIndir | flag(Interface)} } // convertOp: interface -> interface func cvtI2I(v Value, typ Type) Value { if v.IsNil() { ret := Zero(typ) ret.flag |= v.flag & flagRO return ret } return cvtT2I(v.Elem(), typ) } // implemented in ../runtime func chancap(ch unsafe.Pointer) int func chanclose(ch unsafe.Pointer) func chanlen(ch unsafe.Pointer) int // Note: some of the noescape annotations below are technically a lie, // but safe in the context of this package. Functions like chansend // and mapassign don't escape the referent, but may escape anything // the referent points to (they do shallow copies of the referent). // It is safe in this package because the referent may only point // to something a Value may point to, and that is always in the heap // (due to the escapes() call in ValueOf). //go:noescape func chanrecv(t *rtype, ch unsafe.Pointer, nb bool, val unsafe.Pointer) (selected, received bool) //go:noescape func chansend(t *rtype, ch unsafe.Pointer, val unsafe.Pointer, nb bool) bool func makechan(typ *rtype, size uint64) (ch unsafe.Pointer) func makemap(t *rtype) (m unsafe.Pointer) //go:noescape func mapaccess(t *rtype, m unsafe.Pointer, key unsafe.Pointer) (val unsafe.Pointer) //go:noescape func mapassign(t *rtype, m unsafe.Pointer, key, val unsafe.Pointer) //go:noescape func mapdelete(t *rtype, m unsafe.Pointer, key unsafe.Pointer) // m escapes into the return value, but the caller of mapiterinit // doesn't let the return value escape. //go:noescape func mapiterinit(t *rtype, m unsafe.Pointer) unsafe.Pointer //go:noescape func mapiterkey(it unsafe.Pointer) (key unsafe.Pointer) //go:noescape func mapiternext(it unsafe.Pointer) //go:noescape func maplen(m unsafe.Pointer) int // call calls fn with a copy of the n argument bytes pointed at by arg. // After fn returns, reflectcall copies n-retoffset result bytes // back into arg+retoffset before returning. If copying result bytes back, // the caller must pass the argument frame type as argtype, so that // call can execute appropriate write barriers during the copy. func call(argtype *rtype, fn, arg unsafe.Pointer, n uint32, retoffset uint32) func ifaceE2I(t *rtype, src interface{}, dst unsafe.Pointer) // typedmemmove copies a value of type t to dst from src. //go:noescape func typedmemmove(t *rtype, dst, src unsafe.Pointer) // typedmemmovepartial is like typedmemmove but assumes that // dst and src point off bytes into the value and only copies size bytes. //go:noescape func typedmemmovepartial(t *rtype, dst, src unsafe.Pointer, off, size uintptr) // typedslicecopy copies a slice of elemType values from src to dst, // returning the number of elements copied. //go:noescape func typedslicecopy(elemType *rtype, dst, src sliceHeader) int //go:noescape func memclr(ptr unsafe.Pointer, n uintptr) // Dummy annotation marking that the value x escapes, // for use in cases where the reflect code is so clever that // the compiler cannot follow. func escapes(x interface{}) { if dummy.b { dummy.x = x } } var dummy struct { b bool x interface{} } ================================================ FILE: examples/javascript/destructuring.js ================================================ let {a, b} = object let {a, b, ...c} = object const {a, b: {c, d}} = object function a ({b, c}, {d}) {} [a, b] = array; [a, b, ...c] = array; [,, c,, d,] = array; function a({b = true}, [c, d = false]) {} function b({c} = {}) {} ================================================ FILE: examples/javascript/expressions.js ================================================ "A string with \"double\" and 'single' quotes"; 'A string with "double" and \'single\' quotes'; '\\' "\\" 'A string with new \ line'; `one line`; `multi line`; `multi ${2 + 2} hello ${1 + 1, 2 + 2} line`; `$$$$`; `$$$$${ 1 }`; `(a|b)$`; `$`; `$${'$'}$$${'$'}$$$$`; `\ `; `The command \`git ${args.join(' ')}\` exited with an unexpected code: ${exitCode}. The caller should either handle this error, or expect that exit code.` `\\`; `//`; f `hello`; 101; 3.14; 3.14e+1; 0x1ABCDEFabcdef; 0o7632157312; 0b1010101001; 1e+3; theVar; theVar2; $_; var a = b , c = d , e = f; this; null; undefined; true; false; /one\\/; /one/g; /one/i; /one/gim; /on\/e/gim; /on[^/]afe/gim; /[\]/]/; foo ? /* comment */bar : baz var x = {}; var x = { a: "b" }; var x = { c: "d", "e": f, 1: 2 }; var x = { g: h } var x = { [methodName]() { } } x = {a, b, get}; y = {a,}; var x = { foo: true, add(a, b) { return a + b; }, get bar() { return c; }, set bar(a) { c = a; }, *barGenerator() { yield c; }, get() { return 1; } }; var x = { finally() {}, catch() {}, get: function () {}, set: function () {}, static: true, async: true, }; class Foo { static one(a) { return a; }; two(b) { return b; } three(c) { return c; } } class Foo extends require('another-class') { constructor() { super() } bar() { super.a() } } class Foo { catch() {} finally() {} } class Foo { static foo = 2 } @eval class Foo { @foo.bar(baz) static foo() { } } []; [ "item1" ]; [ "item1", ]; [ "item1", item2 ]; [ , item2 ]; [ item2 = 5 ]; [ function() {}, function(arg1, ...arg2) { arg2; }, function stuff() {}, function trailing(a,) {}, function trailing(a,b,) {} ] a => 1; () => 2; (d, e) => 3; (f, g) => { return h; }; (trailing,) => 4; (h, trailing,) => 5; (set, kv) => 2; [ function *() {}, function *generateStuff(arg1, arg2) { yield; yield arg2; } ] function a({b}, c = d, e = f) { } x.someProperty; x[someVariable]; x["some-string"]; return returned.promise() .done( newDefer.resolve ) .fail( newDefer.reject ) return this.map(function (a) { return a.b; }) // a comment .filter(function (c) { return c.d; }) x.someMethod(arg1, "arg2"); var x = function(x, y) { }(a, b); new module.Klass(1, "two"); new Thing; await asyncFunction(); await asyncPromise; async function foo() {} var x = { async bar() { } } class Foo { async bar() {} } async (a) => { return foo; }; i++; i--; i + j * 3 - j % 5; 2 ** i * 3; 2 * i ** 3; +x; -x; i || j; i && j; !a && !b || !c && !d; i >> j; i >>> j; i << j; i & j; i | j; ~i ^ ~j x < y; x <= y; x == y; x === y; x != y; x !== y; x >= y; x > y; x = 0; x.y = 0; x["y"] = 0; async = 0; a = 1, b = 2; c = {d: (3, 4 + 5, 6)}; condition ? case1 : case2; x.y = some.condition ? some.case : some.other.case; typeof x; x instanceof String; delete thing['prop']; true ? delete thing.prop : null; a = void b() s |= 1; t %= 2; w ^= 3; x += 4; y.z *= 5; async += 1; a >>= 1; b >>>= 1; c <<= 1; a <= b && c >= d; a.b = c ? d : e; a && b(c) && d; a && new b(c) && d; typeof a == b && c instanceof d a =
; b = a b c; b = ; a = e {f} g h = {...j} b = b = foo(...rest) (foo - bar) / baz if (foo - bar) /baz/; (this.a() / this.b() - 1) / 2 ⁠// Type definitions for Dexie v1.4.1 // Project: https://github.com/dfahlander/Dexie.js ​// Definitions by: David Fahlander // Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped yield db.users.where('[endpoint+email]') var a = b = c = <> d = e = f = g = h = i = {...children} ================================================ FILE: examples/javascript/literals.js ================================================ 04000 400 100n const últimaVez = 1 var x = { 県: '大阪府', '': '' } "//ok\n//what" ================================================ FILE: examples/javascript/semicolon_insertion.js ================================================ if (a) { var b = c d() e() return f } if (a) d() ++b if (a) d() --b object .someProperty .otherProperty function x() {} return z; a ? b : c a || b a ^ b a !== b a !b; // standalone statement a i; a in b; a ins; a inst; a instanceof b; a instanceofX; if (a) {b} else {c} function a() {b} function c() {return d} var a = new A() .b({c: 'd'}) .e() if (a) { if (b) return c } if (d) { for (;;) break } if (e) { for (f in g) break } if (h) { for (i of j) continue } if (k) { while (l) break } if (m) { do { n; } while (o) } if (p) { var q } function a () { function b () {} function *c () {} class D {} return } let a // comment at end of declaration // comment outside of declaration let b /* comment between declarators */, c /** comment with *stars* **/ /* comment with /slashes/ */ /* third comment in a row */ let d ================================================ FILE: examples/javascript/statements.js ================================================ #!/usr/bin/env node import defaultMember from "module-name"; import * as name from "module-name"; import { member } from "module-name"; import { member1 , member2 } from "module-name"; import { member1 , member2 as alias2 } from "module-name"; import defaultMember, { member1, member2 as alias2 } from "module-name"; import defaultMember, * as name from "module-name"; import "module-name"; import { member1 , member2 as alias2, } from "module-name"; export { name1, name2, name3, nameN }; export { variable1 as name1, variable2 as name2, nameN }; export let name1, name2, nameN; export let name1 = value1, name2 = value2, name3, nameN; export default expression; export default function () { } export default function name1() { } export { name1 as default }; export * from 'foo'; export { name1, name2, nameN } from 'foo'; export { import1 as name1, import2 as name2, nameN } from 'foo'; @injectable() export class Foo { } if (x) log(y); if (a.b) { log(c); d; } if (x) y; else if (a) b; if (a) { c; d; } else { e; } for (var a, b; c; d) e; for (i = 0, init(); i < 10; i++) log(y); for (;;) { z; continue; } for (var i = 0 ; i < l ; i++) { } for (item in items) item(); for (var item in items || {}) item(); for (const {thing} in things) thing(); for (a of b) process(a); for (let {a, b} of items || []) process(a, b); for await (const chunk of stream) { str += chunk; } while (a) b(); do { a; } while (b) do a; while (b) return; return 5; return 1,2; return async; return a; var x = 1; var x, y = {}, z; var x = { // This is a property aProperty: 1, /* * This is a method */ aMethod: function() {} }; // this is the beginning of the script. // here we go. var thing = { // this is a property. // its value is a function. key: function(x /* this is a parameter */) { // this is one statement one(); // this is another statement two(); } }; /* a */ const a = 1; /* b **/ const b = 1; /* c ***/ const c = 1; /* d ***/ const d = 1; y // comment * z; switch (x) { case 1: case 2: something(); break; case "three": somethingElse(); break; default: return 4; } throw new Error("uh oh"); throw f = 1, f; throw g = 2, g try { a; } catch (b) { c; } try { d; } finally { e; } try { f; } catch { g; } finally { h; } if (true) { ; };;; theLoop: for (;;) { if (a) { break theLoop; } else { continue theLoop; } } debugger; debugger with (x) { i; } console.log("HI") ================================================ FILE: examples/ruby/classes.rb ================================================ # Class names must be capitalized. Technically, it's a constant. class Fred # The initialize method is the constructor. The @val is # an object value. def initialize(v) @val = v end # Set it and get it. def set(v) @val = v end def get return @val end end # Objects are created by the new method of the class object. a = Fred.new(10) b = Fred.new(22) print "A: ", a.get, " ", b.get,"\n"; b.set(34) print "B: ", a.get, " ", b.get,"\n"; # Ruby classes are always unfinished works. This does not # re-define Fred, it adds more stuff to it. class Fred def inc @val += 1 end end a.inc b.inc print "C: ", a.get, " ", b.get,"\n"; # Objects may have methods all to themselves. def b.dec @val -= 1 end begin b.dec a.dec rescue StandardError => msg print "Error: ", msg, "\n" end print "D: ", a.get, " ", b.get,"\n"; x = :foo y = :'bar' z = :"doh" require 'uri' begin URI.open('https://google.com') rescue URI::InvalidURIError => e puts "Error: #{e}" end Client.new('test') Client::Subclient.method('test') hash = { key1: 'value2', key2: 'value2' } hash2 = { :key1 => 'value1', :key2 => 'value2' } progress_bar = ProgressBar.create( total: 'test', format: "\e[0;32m%c/%C |%b>%i| %e\e[0m" ) # def and end are the same color def x_to_string x.to_s end # do should use the same color as end in this block of code 10.times do |i| puts i end class Human # A class variable. It is shared by all instances of this class. @@species = 'Homo sapiens' end $global = 'this is a global' @var = "I'm an instance var" defined? @var #=> "instance-variable" defined @var ================================================ FILE: examples/ruby/comments.rb ================================================ # anything else here should be ignored =begin =end =begin whatever =end =begin rdoc =end =begin whatever multiple lines of whatever =end =begin whatever multiple lines of whatever =end # Another comment =begin =e =en =end ================================================ FILE: examples/ruby/control-flow.rb ================================================ while foo do end while foo end while foo do bar end until foo do end until foo do bar end if foo end if foo then else end if true then ;; 123; end if foo then bar else quux end if foo bar elsif quux baz end if foo bar elsif quux baz else bat end unless foo end unless foo then end unless foo else end for x in y do f end for x, y in z do f end for x in y f end for x in y next end for x in y retry end while b break end while b redo end begin end begin foo end begin foo else bar end begin foo ensure bar end begin rescue end begin rescue then end begin rescue bar end begin rescue x end begin rescue x then end begin rescue x bar end begin rescue => x bar end begin rescue x, y bar end begin rescue Error => x end begin rescue Error => x bar end begin rescue *args end foo rescue nil if foo rescue nil elsif bar rescue nil end unless foo rescue nil end begin foo rescue x retry else quux ensure baz end return foo return case foo when bar end case foo when bar else end case key when bar else; leaf end case a when b c when d e else f end case a when *foo c end x = case foo when bar else end x = case foo = bar | baz when bar else end ================================================ FILE: examples/ruby/declarations.rb ================================================ def foo end def foo? end def foo! end def foo bar end def foo= end def `(a) "`" end def -@(a) end def %(a) end def ..(a) end def !~(a) end puts /(/ def /(name) end def / name end def foo super end def foo bar.baz { super } end def foo super.bar a, b end def foo(bar) end def foo(bar); end def foo(bar) end def foo bar end def foo(bar, quux) end def foo bar, quux end def foo(bar: nil, baz:) end def foo(bar = nil) end def foo(bar=nil) end def foo(*options) end def foo(x, *options) end def foo(x, *options, y) end def foo(**options) end def foo(name:, **) end def foo(&block) end def self.foo end def self.foo bar end def self.foo(bar) end def self.foo bar end def self.foo(bar, baz) end def self.foo bar, baz end class Foo end class Foo; end class Foo::Bar end class ::Foo::Bar end class Cß end class Foo < Bar end class Foo < Bar::Quux end class Foo < ::Bar end class Foo < Bar::Baz.new(foo) end class Foo def bar end end class foo()::Bar end class << self end class < 1, :c => 2 x += y x -= y x *= y x **= y x /= y puts "/hi" x ||= y x &&= y x &= y x |= y x %= y x >>= y x <<= y x ^= y a ? b : c a ? b : c true ?")":"c" foo ? true: false foo ? return: false a..b a...b a || b a && b a == b a != b a =~ b a !~ b a < b a <= b a > b a >= b a | b a ^ b a & b a >> b a << b a + b a * b 2+2*2 -a foo -a, bar foo(-a, bar) foo-a @ivar-1 a ** b !a foo foo() print "hello" print("hello") foo a, b, c foo(a, b,) foo(bar(a),) foo.bar foo.bar() foo.bar "hi" foo.bar "hi", 2 foo.bar("hi") foo.bar("hi", 2) foo[bar].() foo.(1, 2) a.() {} a.(b: c) do d end foo.[]() foo&.bar foo(:a => true) foo([] => 1) foo(bar => 1) foo :a => true, :c => 1 foo(a: true) foo a: true foo B: true foo(if: true) foo alias: true foo and: true foo begin: true foo break: true foo case: true foo class: true foo def: true foo defined: true foo do: true foo else: true foo elsif: true foo end: true foo ensure: true foo false: true foo for: true foo if: true foo in: true foo module: true foo next: true foo nil: true foo not: true foo or: true foo redo: true foo rescue: true foo retry: true foo return: true foo self: true foo super: true foo then: true foo true: true foo undef: true foo unless: true foo until: true foo when: true foo while: true foo yield: true foo (b), a foo(&:sort) foo(&bar) foo(&bar, 1) foo &bar foo &bar, 1 foo(*bar) foo *bar foo *%w{ .. lib } foo *(bar.baz) foo :bar, -> (a) { 1 } foo :bar, -> (a) { where(:c => b) } foo :bar, -> (a) { 1 } do end foo(*bar) foo(*[bar, baz].quoz) foo(x, *bar) foo(*bar.baz) foo(**baz) include D::E.f Foo .bar .baz Foo \ .bar foo do |i| foo end foo do |i| i end foo do; end foo(a) do |i| foo end foo.bar a do |i| foo end foo(a) do |name: i, *args| end foo { |i| foo } foo items.any? { |i| i > 0 } foo(bar, baz) { quux } foo { |; i, j| } request.GET -> (d, *f, (x, y)) {} def foo(d, *f, (x, y)) end def foo d, *f, (x, y) end foo do |a, (c, d, *f, (x, y)), *e| end foo [] foo [1] foo[1] lambda {} lambda { foo } lambda(&block) { foo } lambda(&lambda{}) lambda { |foo| 1 } lambda { |a, b, c| 1 2 } lambda { |a, b,| 1 } lambda { |a, b=nil| 1 } lambda { |a, b: nil| 1 } lambda do |foo| 1 end proc = Proc.new lambda = lambda {} proc = proc {} foo \ a, b "abc \ de" foo \ "abc" 10 / 5 h/w "#{foo}" Time.at(timestamp/1000) "#{timestamp}" foo /bar/ foo / bar/ Foo / "bar" "/edit" / a b/ ================================================ FILE: examples/ruby/literals.rb ================================================ :foo :foo! :foo? :foo= :@foo :@foo_0123_bar :@@foo :$foo :$0 :_bar :åäö :+ :- :+@ :-@ :[] :[]= :& :! :` :^ :| :~ :/ :% :* :** :== :=== :=~ :> :>= :>> :< :<= :<< :<=> :'foo bar' :'#{' :"foo bar" :"#" :"foo #{bar}" %s/a/ %s\a\ %s#a# %s{a{b}c} %sc> %s(a(b)c) %s[a[b]c] $foo $$ $! $@ $& $` $' $+ $~ $= $/ $\ $, $; $. $< $> $_ $0 $* $$ $? $: $" $0 $1 $2 $3 $4 $5 $6 $7 $8 $9 $0 $10 $stdin $stdout $stderr $DEBUG $FILENAME $LOAD_PATH $VERBOSE 1234 3.times 1_234 0d1_234 0D1_234 0xa_bcd_ef0_123_456_789 01234567 0o1234567 0B1_0 1.234_5e678_90 1E30 1.0e+6 1.0e-6 -2i +2i 1+1i 1-10i 10+3i 12-34i 2/3r true false nil '' ' ' ' ' '\'' '\\ \n' '\x00\x01\x02' '#{hello' "" " " " " "\"" "\\" "\d" "\#{foo}" "#" "#{foo}" "#{':foo' unless bar}" %q/a/ %q\a\ %q#a# %qc> %q{a{b}c} %q[a[b]c] %q(a(b)c) %/a/ %\a\ %#a# %c> %{a{b}c} %[a[b]c] %(a(b)c) %Q#a# %Q/a/ %Q\a\ %Qc> %Q{a{b}c} %Q[a[b]c] %Q(a(b)c) %q(a) "b" "c" "d" "e" flash[:notice] = "Pattern addition failed for '%s' in '%s'", % [pattern, key] foo("%s '%s' " % [a, b]) ?a ?? ?\n ?\\ ?\377 ?\u{41} ?\M-a ?\C-a ?\M-\C-a ?あ foo(?/) "abc#{ %r(def(ghi#{ `whoami` })klm) }nop" < HTML < { select(<<-SQL) . SQL } <<-ONE `/usr/bin/env test blah blah` `/usr/bin/env test blah \`blah\`` [] [ foo, bar ] [foo, *bar] [foo, *@bar] [foo, *$bar] [foo, :bar => 1] [1, 2].any? { |i| i > 1 } [ foo, ] %w() %w/one two/ %w(word word) %W(a #{b} c) %i() %i/one two/ %i(word word) %I(a #{b} c) %I{ * /#{something}+ ok } {} {:name=>"foo"} { "a" => 1, "b" => 2 } { [] => 1 } { foo => 1 } { alias: :foo, and: :foo, begin: :foo, break: :foo, case: :foo, class: :foo, def: :foo, defined: :foo, do: :foo, else: :foo, elsif: :foo, end: :foo, ensure: :foo, false: :foo, for: :foo, in: :foo, module: :foo, next: :foo, nil: :foo, not: :foo, or: :foo, redo: :foo, rescue: :foo, retry: :foo, return: :foo, self: :foo, super: :foo, then: :foo, true: :foo, undef: :foo, when: :foo, yield: :foo, if: :foo, unless: :foo, while: :foo, until: :foo } { a: 1, b: 2, "c": 3 } {a:1, b:2, "c":3 } { a: 1, } {a: 1, **{b: 2}} { :pusher => pusher, # Only warm caches if there are fewer than 10 tags and branches. :should_warm_caches_after => 10, } /^(foo|bar[^_])$/i /word#{foo}word/ /word#word/ /#/ %r/a/ %r\a\ %r#a# %rc> %r{a{b}c} %r[a[b]c] %r(a(b)c) %r(#) %r/a#{b}c/ %r(a#{b}c) -> {} -> { foo } -> foo { 1 } -> (foo) { 1 } -> *foo { 1 } -> foo: 1 { 2 } -> foo, bar { 2 } -> (a, b, c) { 1 2 } -> (foo) do 1 end Cß @äö @@äö :äö äö ================================================ FILE: examples/ruby/statements.rb ================================================ foo if bar return if false return true if foo return nil if foo foo while bar foo unless bar foo until bar alias :foo :bar alias foo bar alias $FOO $& alias foo + undef :foo undef foo undef + undef :foo, :bar ================================================ FILE: examples/rust/ast.rs ================================================ // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // // Licensed under the Apache License, Version 2.0 or the MIT license // , at your // option. This file may not be copied, modified, or distributed // except according to those terms. // The Rust abstract syntax tree. pub use self::TyParamBound::*; pub use self::UnsafeSource::*; pub use self::PathParameters::*; pub use symbol::{Ident, Symbol as Name}; pub use util::ThinVec; pub use util::parser::ExprPrecedence; use syntax_pos::{Span, DUMMY_SP}; use codemap::{respan, Spanned}; use abi::Abi; use ext::hygiene::{Mark, SyntaxContext}; use print::pprust; use ptr::P; use rustc_data_structures::indexed_vec; use symbol::{Symbol, keywords}; use tokenstream::{ThinTokenStream, TokenStream}; use serialize::{self, Encoder, Decoder}; use std::collections::HashSet; use std::fmt; use std::rc::Rc; use std::u32; #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Copy)] pub struct Lifetime { pub id: NodeId, pub span: Span, pub ident: Ident, } impl fmt::Debug for Lifetime { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "lifetime({}: {})", self.id, pprust::lifetime_to_string(self)) } } /// A lifetime definition, e.g. `'a: 'b+'c+'d` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct LifetimeDef { pub attrs: ThinVec, pub lifetime: Lifetime, pub bounds: Vec } /// A "Path" is essentially Rust's notion of a name. /// /// It's represented as a sequence of identifiers, /// along with a bunch of supporting information. /// /// E.g. `std::cmp::PartialEq` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] pub struct Path { pub span: Span, /// The segments in the path: the things separated by `::`. /// Global paths begin with `keywords::CrateRoot`. pub segments: Vec, } impl<'a> PartialEq<&'a str> for Path { fn eq(&self, string: &&'a str) -> bool { self.segments.len() == 1 && self.segments[0].identifier.name == *string } } impl fmt::Debug for Path { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "path({})", pprust::path_to_string(self)) } } impl fmt::Display for Path { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", pprust::path_to_string(self)) } } impl Path { // convert a span and an identifier to the corresponding // 1-segment path pub fn from_ident(s: Span, identifier: Ident) -> Path { Path { span: s, segments: vec![PathSegment::from_ident(identifier, s)], } } // Add starting "crate root" segment to all paths except those that // already have it or start with `self`, `super`, `Self` or `$crate`. pub fn default_to_global(mut self) -> Path { if !self.is_global() { let ident = self.segments[0].identifier; if !::parse::token::Ident(ident).is_path_segment_keyword() || ident.name == keywords::Crate.name() { self.segments.insert(0, PathSegment::crate_root(self.span)); } } self } pub fn is_global(&self) -> bool { !self.segments.is_empty() && self.segments[0].identifier.name == keywords::CrateRoot.name() } } /// A segment of a path: an identifier, an optional lifetime, and a set of types. /// /// E.g. `std`, `String` or `Box` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct PathSegment { /// The identifier portion of this path segment. pub identifier: Ident, /// Span of the segment identifier. pub span: Span, /// Type/lifetime parameters attached to this path. They come in /// two flavors: `Path` and `Path(A,B) -> C`. /// `None` means that no parameter list is supplied (`Path`), /// `Some` means that parameter list is supplied (`Path`) /// but it can be empty (`Path<>`). /// `P` is used as a size optimization for the common case with no parameters. pub parameters: Option>, } impl PathSegment { pub fn from_ident(ident: Ident, span: Span) -> Self { PathSegment { identifier: ident, span: span, parameters: None } } pub fn crate_root(span: Span) -> Self { PathSegment { identifier: Ident { ctxt: span.ctxt(), ..keywords::CrateRoot.ident() }, span, parameters: None, } } } /// Parameters of a path segment. /// /// E.g. `` as in `Foo` or `(A, B)` as in `Foo(A, B)` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum PathParameters { /// The `<'a, A,B,C>` in `foo::bar::baz::<'a, A,B,C>` AngleBracketed(AngleBracketedParameterData), /// The `(A,B)` and `C` in `Foo(A,B) -> C` Parenthesized(ParenthesizedParameterData), } impl PathParameters { pub fn span(&self) -> Span { match *self { AngleBracketed(ref data) => data.span, Parenthesized(ref data) => data.span, } } } /// A path like `Foo<'a, T>` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Default)] pub struct AngleBracketedParameterData { /// Overall span pub span: Span, /// The lifetime parameters for this path segment. pub lifetimes: Vec, /// The type parameters for this path segment, if present. pub types: Vec>, /// Bindings (equality constraints) on associated types, if present. /// /// E.g., `Foo`. pub bindings: Vec, } impl Into>> for AngleBracketedParameterData { fn into(self) -> Option> { Some(P(PathParameters::AngleBracketed(self))) } } impl Into>> for ParenthesizedParameterData { fn into(self) -> Option> { Some(P(PathParameters::Parenthesized(self))) } } /// A path like `Foo(A,B) -> C` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct ParenthesizedParameterData { /// Overall span pub span: Span, /// `(A,B)` pub inputs: Vec>, /// `C` pub output: Option>, } #[derive(Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Hash, Debug)] pub struct NodeId(u32); impl NodeId { pub fn new(x: usize) -> NodeId { assert!(x < (u32::MAX as usize)); NodeId(x as u32) } pub fn from_u32(x: u32) -> NodeId { NodeId(x) } pub fn as_usize(&self) -> usize { self.0 as usize } pub fn as_u32(&self) -> u32 { self.0 } pub fn placeholder_from_mark(mark: Mark) -> Self { NodeId(mark.as_u32()) } pub fn placeholder_to_mark(self) -> Mark { Mark::from_u32(self.0) } } impl fmt::Display for NodeId { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Display::fmt(&self.0, f) } } impl serialize::UseSpecializedEncodable for NodeId { fn default_encode(&self, s: &mut S) -> Result<(), S::Error> { s.emit_u32(self.0) } } impl serialize::UseSpecializedDecodable for NodeId { fn default_decode(d: &mut D) -> Result { d.read_u32().map(NodeId) } } impl indexed_vec::Idx for NodeId { fn new(idx: usize) -> Self { NodeId::new(idx) } fn index(self) -> usize { self.as_usize() } } /// Node id used to represent the root of the crate. pub const CRATE_NODE_ID: NodeId = NodeId(0); /// When parsing and doing expansions, we initially give all AST nodes this AST /// node value. Then later, in the renumber pass, we renumber them to have /// small, positive ids. pub const DUMMY_NODE_ID: NodeId = NodeId(!0); /// The AST represents all type param bounds as types. /// typeck::collect::compute_bounds matches these against /// the "special" built-in traits (see middle::lang_items) and /// detects Copy, Send and Sync. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum TyParamBound { TraitTyParamBound(PolyTraitRef, TraitBoundModifier), RegionTyParamBound(Lifetime) } /// A modifier on a bound, currently this is only used for `?Sized`, where the /// modifier is `Maybe`. Negative bounds should also be handled here. #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum TraitBoundModifier { None, Maybe, } pub type TyParamBounds = Vec; #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct TyParam { pub attrs: ThinVec, pub ident: Ident, pub id: NodeId, pub bounds: TyParamBounds, pub default: Option>, pub span: Span, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum GenericParam { Lifetime(LifetimeDef), Type(TyParam), } impl GenericParam { pub fn is_lifetime_param(&self) -> bool { match *self { GenericParam::Lifetime(_) => true, _ => false, } } pub fn is_type_param(&self) -> bool { match *self { GenericParam::Type(_) => true, _ => false, } } } /// Represents lifetime, type and const parameters attached to a declaration of /// a function, enum, trait, etc. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Generics { pub params: Vec, pub where_clause: WhereClause, pub span: Span, } impl Generics { pub fn is_lt_parameterized(&self) -> bool { self.params.iter().any(|param| param.is_lifetime_param()) } pub fn is_type_parameterized(&self) -> bool { self.params.iter().any(|param| param.is_type_param()) } pub fn is_parameterized(&self) -> bool { !self.params.is_empty() } pub fn span_for_name(&self, name: &str) -> Option { for param in &self.params { if let GenericParam::Type(ref t) = *param { if t.ident.name == name { return Some(t.span); } } } None } } impl Default for Generics { /// Creates an instance of `Generics`. fn default() -> Generics { Generics { params: Vec::new(), where_clause: WhereClause { id: DUMMY_NODE_ID, predicates: Vec::new(), span: DUMMY_SP, }, span: DUMMY_SP, } } } /// A `where` clause in a definition #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct WhereClause { pub id: NodeId, pub predicates: Vec, pub span: Span, } /// A single predicate in a `where` clause #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum WherePredicate { /// A type binding, e.g. `for<'c> Foo: Send+Clone+'c` BoundPredicate(WhereBoundPredicate), /// A lifetime predicate, e.g. `'a: 'b+'c` RegionPredicate(WhereRegionPredicate), /// An equality predicate (unsupported) EqPredicate(WhereEqPredicate), } /// A type bound. /// /// E.g. `for<'c> Foo: Send+Clone+'c` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct WhereBoundPredicate { pub span: Span, /// Any generics from a `for` binding pub bound_generic_params: Vec, /// The type being bounded pub bounded_ty: P, /// Trait and lifetime bounds (`Clone+Send+'static`) pub bounds: TyParamBounds, } /// A lifetime predicate. /// /// E.g. `'a: 'b+'c` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct WhereRegionPredicate { pub span: Span, pub lifetime: Lifetime, pub bounds: Vec, } /// An equality predicate (unsupported). /// /// E.g. `T=int` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct WhereEqPredicate { pub id: NodeId, pub span: Span, pub lhs_ty: P, pub rhs_ty: P, } /// The set of MetaItems that define the compilation environment of the crate, /// used to drive conditional compilation pub type CrateConfig = HashSet<(Name, Option)>; #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Crate { pub module: Mod, pub attrs: Vec, pub span: Span, } /// A spanned compile-time attribute list item. pub type NestedMetaItem = Spanned; /// Possible values inside of compile-time attribute lists. /// /// E.g. the '..' in `#[name(..)]`. #[derive(Clone, Eq, RustcEncodable, RustcDecodable, Hash, Debug, PartialEq)] pub enum NestedMetaItemKind { /// A full MetaItem, for recursive meta items. MetaItem(MetaItem), /// A literal. /// /// E.g. "foo", 64, true Literal(Lit), } /// A spanned compile-time attribute item. /// /// E.g. `#[test]`, `#[derive(..)]` or `#[feature = "foo"]` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct MetaItem { pub name: Name, pub node: MetaItemKind, pub span: Span, } /// A compile-time attribute item. /// /// E.g. `#[test]`, `#[derive(..)]` or `#[feature = "foo"]` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum MetaItemKind { /// Word meta item. /// /// E.g. `test` as in `#[test]` Word, /// List meta item. /// /// E.g. `derive(..)` as in `#[derive(..)]` List(Vec), /// Name value meta item. /// /// E.g. `feature = "foo"` as in `#[feature = "foo"]` NameValue(Lit) } /// A Block (`{ .. }`). /// /// E.g. `{ .. }` as in `fn foo() { .. }` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Block { /// Statements in a block pub stmts: Vec, pub id: NodeId, /// Distinguishes between `unsafe { ... }` and `{ ... }` pub rules: BlockCheckMode, pub span: Span, pub recovered: bool, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] pub struct Pat { pub id: NodeId, pub node: PatKind, pub span: Span, } impl fmt::Debug for Pat { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "pat({}: {})", self.id, pprust::pat_to_string(self)) } } impl Pat { pub(super) fn to_ty(&self) -> Option> { let node = match &self.node { PatKind::Wild => TyKind::Infer, PatKind::Ident(BindingMode::ByValue(Mutability::Immutable), ident, None) => TyKind::Path(None, Path::from_ident(ident.span, ident.node)), PatKind::Path(qself, path) => TyKind::Path(qself.clone(), path.clone()), PatKind::Mac(mac) => TyKind::Mac(mac.clone()), PatKind::Ref(pat, mutbl) => pat.to_ty().map(|ty| TyKind::Rptr(None, MutTy { ty, mutbl: *mutbl }))?, PatKind::Slice(pats, None, _) if pats.len() == 1 => pats[0].to_ty().map(TyKind::Slice)?, PatKind::Tuple(pats, None) => { let mut tys = Vec::new(); for pat in pats { tys.push(pat.to_ty()?); } TyKind::Tup(tys) } _ => return None, }; Some(P(Ty { node, id: self.id, span: self.span })) } pub fn walk(&self, it: &mut F) -> bool where F: FnMut(&Pat) -> bool { if !it(self) { return false; } match self.node { PatKind::Ident(_, _, Some(ref p)) => p.walk(it), PatKind::Struct(_, ref fields, _) => { fields.iter().all(|field| field.node.pat.walk(it)) } PatKind::TupleStruct(_, ref s, _) | PatKind::Tuple(ref s, _) => { s.iter().all(|p| p.walk(it)) } PatKind::Box(ref s) | PatKind::Ref(ref s, _) => { s.walk(it) } PatKind::Slice(ref before, ref slice, ref after) => { before.iter().all(|p| p.walk(it)) && slice.iter().all(|p| p.walk(it)) && after.iter().all(|p| p.walk(it)) } PatKind::Wild | PatKind::Lit(_) | PatKind::Range(..) | PatKind::Ident(..) | PatKind::Path(..) | PatKind::Mac(_) => { true } } } } /// A single field in a struct pattern /// /// Patterns like the fields of Foo `{ x, ref y, ref mut z }` /// are treated the same as` x: x, y: ref y, z: ref mut z`, /// except is_shorthand is true #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct FieldPat { /// The identifier for the field pub ident: Ident, /// The pattern the field is destructured to pub pat: P, pub is_shorthand: bool, pub attrs: ThinVec, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum BindingMode { ByRef(Mutability), ByValue(Mutability), } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum RangeEnd { Included(RangeSyntax), Excluded, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum RangeSyntax { DotDotDot, DotDotEq, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum PatKind { /// Represents a wildcard pattern (`_`) Wild, /// A `PatKind::Ident` may either be a new bound variable (`ref mut binding @ OPT_SUBPATTERN`), /// or a unit struct/variant pattern, or a const pattern (in the last two cases the third /// field must be `None`). Disambiguation cannot be done with parser alone, so it happens /// during name resolution. Ident(BindingMode, SpannedIdent, Option>), /// A struct or struct variant pattern, e.g. `Variant {x, y, ..}`. /// The `bool` is `true` in the presence of a `..`. Struct(Path, Vec>, bool), /// A tuple struct/variant pattern `Variant(x, y, .., z)`. /// If the `..` pattern fragment is present, then `Option` denotes its position. /// 0 <= position <= subpats.len() TupleStruct(Path, Vec>, Option), /// A possibly qualified path pattern. /// Unqualified path patterns `A::B::C` can legally refer to variants, structs, constants /// or associated constants. Qualified path patterns `::B::C`/`::B::C` can /// only legally refer to associated constants. Path(Option, Path), /// A tuple pattern `(a, b)`. /// If the `..` pattern fragment is present, then `Option` denotes its position. /// 0 <= position <= subpats.len() Tuple(Vec>, Option), /// A `box` pattern Box(P), /// A reference pattern, e.g. `&mut (a, b)` Ref(P, Mutability), /// A literal Lit(P), /// A range pattern, e.g. `1...2`, `1..=2` or `1..2` Range(P, P, RangeEnd), /// `[a, b, ..i, y, z]` is represented as: /// `PatKind::Slice(box [a, b], Some(i), box [y, z])` Slice(Vec>, Option>, Vec>), /// A macro pattern; pre-expansion Mac(Mac), } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum Mutability { Mutable, Immutable, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum BinOpKind { /// The `+` operator (addition) Add, /// The `-` operator (subtraction) Sub, /// The `*` operator (multiplication) Mul, /// The `/` operator (division) Div, /// The `%` operator (modulus) Rem, /// The `&&` operator (logical and) And, /// The `||` operator (logical or) Or, /// The `^` operator (bitwise xor) BitXor, /// The `&` operator (bitwise and) BitAnd, /// The `|` operator (bitwise or) BitOr, /// The `<<` operator (shift left) Shl, /// The `>>` operator (shift right) Shr, /// The `==` operator (equality) Eq, /// The `<` operator (less than) Lt, /// The `<=` operator (less than or equal to) Le, /// The `!=` operator (not equal to) Ne, /// The `>=` operator (greater than or equal to) Ge, /// The `>` operator (greater than) Gt, } impl BinOpKind { pub fn to_string(&self) -> &'static str { use self::BinOpKind::*; match *self { Add => "+", Sub => "-", Mul => "*", Div => "/", Rem => "%", And => "&&", Or => "||", BitXor => "^", BitAnd => "&", BitOr => "|", Shl => "<<", Shr => ">>", Eq => "==", Lt => "<", Le => "<=", Ne => "!=", Ge => ">=", Gt => ">", } } pub fn lazy(&self) -> bool { match *self { BinOpKind::And | BinOpKind::Or => true, _ => false } } pub fn is_shift(&self) -> bool { match *self { BinOpKind::Shl | BinOpKind::Shr => true, _ => false } } pub fn is_comparison(&self) -> bool { use self::BinOpKind::*; match *self { Eq | Lt | Le | Ne | Gt | Ge => true, And | Or | Add | Sub | Mul | Div | Rem | BitXor | BitAnd | BitOr | Shl | Shr => false, } } /// Returns `true` if the binary operator takes its arguments by value pub fn is_by_value(&self) -> bool { !self.is_comparison() } } pub type BinOp = Spanned; #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum UnOp { /// The `*` operator for dereferencing Deref, /// The `!` operator for logical inversion Not, /// The `-` operator for negation Neg, } impl UnOp { /// Returns `true` if the unary operator takes its argument by value pub fn is_by_value(u: UnOp) -> bool { match u { UnOp::Neg | UnOp::Not => true, _ => false, } } pub fn to_string(op: UnOp) -> &'static str { match op { UnOp::Deref => "*", UnOp::Not => "!", UnOp::Neg => "-", } } } /// A statement #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] pub struct Stmt { pub id: NodeId, pub node: StmtKind, pub span: Span, } impl Stmt { pub fn add_trailing_semicolon(mut self) -> Self { self.node = match self.node { StmtKind::Expr(expr) => StmtKind::Semi(expr), StmtKind::Mac(mac) => StmtKind::Mac(mac.map(|(mac, _style, attrs)| { (mac, MacStmtStyle::Semicolon, attrs) })), node => node, }; self } pub fn is_item(&self) -> bool { match self.node { StmtKind::Local(_) => true, _ => false, } } } impl fmt::Debug for Stmt { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "stmt({}: {})", self.id.to_string(), pprust::stmt_to_string(self)) } } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] pub enum StmtKind { /// A local (let) binding. Local(P), /// An item definition. Item(P), /// Expr without trailing semi-colon. Expr(P), /// Expr with a trailing semi-colon. Semi(P), /// Macro. Mac(P<(Mac, MacStmtStyle, ThinVec)>), } #[derive(Clone, Copy, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum MacStmtStyle { /// The macro statement had a trailing semicolon, e.g. `foo! { ... };` /// `foo!(...);`, `foo![...];` Semicolon, /// The macro statement had braces; e.g. foo! { ... } Braces, /// The macro statement had parentheses or brackets and no semicolon; e.g. /// `foo!(...)`. All of these will end up being converted into macro /// expressions. NoBraces, } /// Local represents a `let` statement, e.g., `let : = ;` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Local { pub pat: P, pub ty: Option>, /// Initializer expression to set the value, if any pub init: Option>, pub id: NodeId, pub span: Span, pub attrs: ThinVec, } /// An arm of a 'match'. /// /// E.g. `0...10 => { println!("match!") }` as in /// /// ``` /// match 123 { /// 0...10 => { println!("match!") }, /// _ => { println!("no match!") }, /// } /// ``` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Arm { pub attrs: Vec, pub pats: Vec>, pub guard: Option>, pub body: P, pub beginning_vert: Option, // For RFC 1925 feature gate } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Field { pub ident: SpannedIdent, pub expr: P, pub span: Span, pub is_shorthand: bool, pub attrs: ThinVec, } pub type SpannedIdent = Spanned; #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum BlockCheckMode { Default, Unsafe(UnsafeSource), } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum UnsafeSource { CompilerGenerated, UserProvided, } /// An expression #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash,)] pub struct Expr { pub id: NodeId, pub node: ExprKind, pub span: Span, pub attrs: ThinVec } impl Expr { /// Wether this expression would be valid somewhere that expects a value, for example, an `if` /// condition. pub fn returns(&self) -> bool { if let ExprKind::Block(ref block) = self.node { match block.stmts.last().map(|last_stmt| &last_stmt.node) { // implicit return Some(&StmtKind::Expr(_)) => true, Some(&StmtKind::Semi(ref expr)) => { if let ExprKind::Ret(_) = expr.node { // last statement is explicit return true } else { false } } // This is a block that doesn't end in either an implicit or explicit return _ => false, } } else { // This is not a block, it is a value true } } fn to_bound(&self) -> Option { match &self.node { ExprKind::Path(None, path) => Some(TraitTyParamBound(PolyTraitRef::new(Vec::new(), path.clone(), self.span), TraitBoundModifier::None)), _ => None, } } pub(super) fn to_ty(&self) -> Option> { let node = match &self.node { ExprKind::Path(qself, path) => TyKind::Path(qself.clone(), path.clone()), ExprKind::Mac(mac) => TyKind::Mac(mac.clone()), ExprKind::Paren(expr) => expr.to_ty().map(TyKind::Paren)?, ExprKind::AddrOf(mutbl, expr) => expr.to_ty().map(|ty| TyKind::Rptr(None, MutTy { ty, mutbl: *mutbl }))?, ExprKind::Repeat(expr, expr_len) => expr.to_ty().map(|ty| TyKind::Array(ty, expr_len.clone()))?, ExprKind::Array(exprs) if exprs.len() == 1 => exprs[0].to_ty().map(TyKind::Slice)?, ExprKind::Tup(exprs) => { let mut tys = Vec::new(); for expr in exprs { tys.push(expr.to_ty()?); } TyKind::Tup(tys) } ExprKind::Binary(binop, lhs, rhs) if binop.node == BinOpKind::Add => if let (Some(lhs), Some(rhs)) = (lhs.to_bound(), rhs.to_bound()) { TyKind::TraitObject(vec![lhs, rhs], TraitObjectSyntax::None) } else { return None; } _ => return None, }; Some(P(Ty { node, id: self.id, span: self.span })) } pub fn precedence(&self) -> ExprPrecedence { match self.node { ExprKind::Box(_) => ExprPrecedence::Box, ExprKind::InPlace(..) => ExprPrecedence::InPlace, ExprKind::Array(_) => ExprPrecedence::Array, ExprKind::Call(..) => ExprPrecedence::Call, ExprKind::MethodCall(..) => ExprPrecedence::MethodCall, ExprKind::Tup(_) => ExprPrecedence::Tup, ExprKind::Binary(op, ..) => ExprPrecedence::Binary(op.node), ExprKind::Unary(..) => ExprPrecedence::Unary, ExprKind::Lit(_) => ExprPrecedence::Lit, ExprKind::Type(..) | ExprKind::Cast(..) => ExprPrecedence::Cast, ExprKind::If(..) => ExprPrecedence::If, ExprKind::IfLet(..) => ExprPrecedence::IfLet, ExprKind::While(..) => ExprPrecedence::While, ExprKind::WhileLet(..) => ExprPrecedence::WhileLet, ExprKind::ForLoop(..) => ExprPrecedence::ForLoop, ExprKind::Loop(..) => ExprPrecedence::Loop, ExprKind::Match(..) => ExprPrecedence::Match, ExprKind::Closure(..) => ExprPrecedence::Closure, ExprKind::Block(..) => ExprPrecedence::Block, ExprKind::Catch(..) => ExprPrecedence::Catch, ExprKind::Assign(..) => ExprPrecedence::Assign, ExprKind::AssignOp(..) => ExprPrecedence::AssignOp, ExprKind::Field(..) => ExprPrecedence::Field, ExprKind::TupField(..) => ExprPrecedence::TupField, ExprKind::Index(..) => ExprPrecedence::Index, ExprKind::Range(..) => ExprPrecedence::Range, ExprKind::Path(..) => ExprPrecedence::Path, ExprKind::AddrOf(..) => ExprPrecedence::AddrOf, ExprKind::Break(..) => ExprPrecedence::Break, ExprKind::Continue(..) => ExprPrecedence::Continue, ExprKind::Ret(..) => ExprPrecedence::Ret, ExprKind::InlineAsm(..) => ExprPrecedence::InlineAsm, ExprKind::Mac(..) => ExprPrecedence::Mac, ExprKind::Struct(..) => ExprPrecedence::Struct, ExprKind::Repeat(..) => ExprPrecedence::Repeat, ExprKind::Paren(..) => ExprPrecedence::Paren, ExprKind::Try(..) => ExprPrecedence::Try, ExprKind::Yield(..) => ExprPrecedence::Yield, } } } impl fmt::Debug for Expr { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "expr({}: {})", self.id, pprust::expr_to_string(self)) } } /// Limit types of a range (inclusive or exclusive) #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum RangeLimits { /// Inclusive at the beginning, exclusive at the end HalfOpen, /// Inclusive at the beginning and end Closed, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum ExprKind { /// A `box x` expression. Box(P), /// First expr is the place; second expr is the value. InPlace(P, P), /// An array (`[a, b, c, d]`) Array(Vec>), /// A function call /// /// The first field resolves to the function itself, /// and the second field is the list of arguments. /// This also represents calling the constructor of /// tuple-like ADTs such as tuple structs and enum variants. Call(P, Vec>), /// A method call (`x.foo::<'static, Bar, Baz>(a, b, c, d)`) /// /// The `PathSegment` represents the method name and its generic arguments /// (within the angle brackets). /// The first element of the vector of `Expr`s is the expression that evaluates /// to the object on which the method is being called on (the receiver), /// and the remaining elements are the rest of the arguments. /// Thus, `x.foo::(a, b, c, d)` is represented as /// `ExprKind::MethodCall(PathSegment { foo, [Bar, Baz] }, [x, a, b, c, d])`. MethodCall(PathSegment, Vec>), /// A tuple (`(a, b, c ,d)`) Tup(Vec>), /// A binary operation (For example: `a + b`, `a * b`) Binary(BinOp, P, P), /// A unary operation (For example: `!x`, `*x`) Unary(UnOp, P), /// A literal (For example: `1`, `"foo"`) Lit(P), /// A cast (`foo as f64`) Cast(P, P), Type(P, P), /// An `if` block, with an optional else block /// /// `if expr { block } else { expr }` If(P, P, Option>), /// An `if let` expression with an optional else block /// /// `if let pat = expr { block } else { expr }` /// /// This is desugared to a `match` expression. IfLet(P, P, P, Option>), /// A while loop, with an optional label /// /// `'label: while expr { block }` While(P, P, Option), /// A while-let loop, with an optional label /// /// `'label: while let pat = expr { block }` /// /// This is desugared to a combination of `loop` and `match` expressions. WhileLet(P, P, P, Option), /// A for loop, with an optional label /// /// `'label: for pat in expr { block }` /// /// This is desugared to a combination of `loop` and `match` expressions. ForLoop(P, P, P, Option), /// Conditionless loop (can be exited with break, continue, or return) /// /// `'label: loop { block }` Loop(P, Option), /// A `match` block. Match(P, Vec), /// A closure (for example, `move |a, b, c| a + b + c`) /// /// The final span is the span of the argument block `|...|` Closure(CaptureBy, P, P, Span), /// A block (`{ ... }`) Block(P), /// A catch block (`catch { ... }`) Catch(P), /// An assignment (`a = foo()`) Assign(P, P), /// An assignment with an operator /// /// For example, `a += 1`. AssignOp(BinOp, P, P), /// Access of a named struct field (`obj.foo`) Field(P, SpannedIdent), /// Access of an unnamed field of a struct or tuple-struct /// /// For example, `foo.0`. TupField(P, Spanned), /// An indexing operation (`foo[2]`) Index(P, P), /// A range (`1..2`, `1..`, `..2`, `1...2`, `1...`, `...2`) Range(Option>, Option>, RangeLimits), /// Variable reference, possibly containing `::` and/or type /// parameters, e.g. foo::bar::. /// /// Optionally "qualified", /// E.g. ` as SomeTrait>::SomeType`. Path(Option, Path), /// A referencing operation (`&a` or `&mut a`) AddrOf(Mutability, P), /// A `break`, with an optional label to break, and an optional expression Break(Option, Option>), /// A `continue`, with an optional label Continue(Option), /// A `return`, with an optional value to be returned Ret(Option>), /// Output of the `asm!()` macro InlineAsm(P), /// A macro invocation; pre-expansion Mac(Mac), /// A struct literal expression. /// /// For example, `Foo {x: 1, y: 2}`, or /// `Foo {x: 1, .. base}`, where `base` is the `Option`. Struct(Path, Vec, Option>), /// An array literal constructed from one repeated element. /// /// For example, `[1; 5]`. The first expression is the element /// to be repeated; the second is the number of times to repeat it. Repeat(P, P), /// No-op: used solely so we can pretty-print faithfully Paren(P), /// `expr?` Try(P), /// A `yield`, with an optional value to be yielded Yield(Option>), } /// The explicit Self type in a "qualified path". The actual /// path, including the trait and the associated item, is stored /// separately. `position` represents the index of the associated /// item qualified with this Self type. /// /// ```ignore (only-for-syntax-highlight) /// as a::b::Trait>::AssociatedItem /// ^~~~~ ~~~~~~~~~~~~~~^ /// ty position = 3 /// /// >::AssociatedItem /// ^~~~~ ^ /// ty position = 0 /// ``` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct QSelf { pub ty: P, pub position: usize } /// A capture clause #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum CaptureBy { Value, Ref, } pub type Mac = Spanned; /// Represents a macro invocation. The Path indicates which macro /// is being invoked, and the vector of token-trees contains the source /// of the macro invocation. /// /// NB: the additional ident for a macro_rules-style macro is actually /// stored in the enclosing item. Oog. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Mac_ { pub path: Path, pub tts: ThinTokenStream, } impl Mac_ { pub fn stream(&self) -> TokenStream { self.tts.clone().into() } } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct MacroDef { pub tokens: ThinTokenStream, pub legacy: bool, } impl MacroDef { pub fn stream(&self) -> TokenStream { self.tokens.clone().into() } } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum StrStyle { /// A regular string, like `"foo"` Cooked, /// A raw string, like `r##"foo"##` /// /// The uint is the number of `#` symbols used Raw(usize) } /// A literal pub type Lit = Spanned; #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum LitIntType { Signed(IntTy), Unsigned(UintTy), Unsuffixed, } /// Literal kind. /// /// E.g. `"foo"`, `42`, `12.34` or `bool` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum LitKind { /// A string literal (`"foo"`) Str(Symbol, StrStyle), /// A byte string (`b"foo"`) ByteStr(Rc>), /// A byte char (`b'f'`) Byte(u8), /// A character literal (`'a'`) Char(char), /// An integer literal (`1`) Int(u128, LitIntType), /// A float literal (`1f64` or `1E10f64`) Float(Symbol, FloatTy), /// A float literal without a suffix (`1.0 or 1.0E10`) FloatUnsuffixed(Symbol), /// A boolean literal Bool(bool), } impl LitKind { /// Returns true if this literal is a string and false otherwise. pub fn is_str(&self) -> bool { match *self { LitKind::Str(..) => true, _ => false, } } /// Returns true if this literal has no suffix. Note: this will return true /// for literals with prefixes such as raw strings and byte strings. pub fn is_unsuffixed(&self) -> bool { match *self { // unsuffixed variants LitKind::Str(..) | LitKind::ByteStr(..) | LitKind::Byte(..) | LitKind::Char(..) | LitKind::Int(_, LitIntType::Unsuffixed) | LitKind::FloatUnsuffixed(..) | LitKind::Bool(..) => true, // suffixed variants LitKind::Int(_, LitIntType::Signed(..)) | LitKind::Int(_, LitIntType::Unsigned(..)) | LitKind::Float(..) => false, } } /// Returns true if this literal has a suffix. pub fn is_suffixed(&self) -> bool { !self.is_unsuffixed() } } // NB: If you change this, you'll probably want to change the corresponding // type structure in middle/ty.rs as well. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct MutTy { pub ty: P, pub mutbl: Mutability, } /// Represents a method's signature in a trait declaration, /// or in an implementation. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct MethodSig { pub unsafety: Unsafety, pub constness: Spanned, pub abi: Abi, pub decl: P, } /// Represents an item declaration within a trait declaration, /// possibly including a default implementation. A trait item is /// either required (meaning it doesn't have an implementation, just a /// signature) or provided (meaning it has a default implementation). #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct TraitItem { pub id: NodeId, pub ident: Ident, pub attrs: Vec, pub generics: Generics, pub node: TraitItemKind, pub span: Span, /// See `Item::tokens` for what this is pub tokens: Option, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum TraitItemKind { Const(P, Option>), Method(MethodSig, Option>), Type(TyParamBounds, Option>), Macro(Mac), } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct ImplItem { pub id: NodeId, pub ident: Ident, pub vis: Visibility, pub defaultness: Defaultness, pub attrs: Vec, pub generics: Generics, pub node: ImplItemKind, pub span: Span, /// See `Item::tokens` for what this is pub tokens: Option, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum ImplItemKind { Const(P, P), Method(MethodSig, P), Type(P), Macro(Mac), } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Copy, PartialOrd, Ord)] pub enum IntTy { Isize, I8, I16, I32, I64, I128, } impl fmt::Debug for IntTy { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Display::fmt(self, f) } } impl fmt::Display for IntTy { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.ty_to_string()) } } impl IntTy { pub fn ty_to_string(&self) -> &'static str { match *self { IntTy::Isize => "isize", IntTy::I8 => "i8", IntTy::I16 => "i16", IntTy::I32 => "i32", IntTy::I64 => "i64", IntTy::I128 => "i128", } } pub fn val_to_string(&self, val: i128) -> String { // cast to a u128 so we can correctly print INT128_MIN. All integral types // are parsed as u128, so we wouldn't want to print an extra negative // sign. format!("{}{}", val as u128, self.ty_to_string()) } pub fn bit_width(&self) -> Option { Some(match *self { IntTy::Isize => return None, IntTy::I8 => 8, IntTy::I16 => 16, IntTy::I32 => 32, IntTy::I64 => 64, IntTy::I128 => 128, }) } } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Copy, PartialOrd, Ord)] pub enum UintTy { Usize, U8, U16, U32, U64, U128, } impl UintTy { pub fn ty_to_string(&self) -> &'static str { match *self { UintTy::Usize => "usize", UintTy::U8 => "u8", UintTy::U16 => "u16", UintTy::U32 => "u32", UintTy::U64 => "u64", UintTy::U128 => "u128", } } pub fn val_to_string(&self, val: u128) -> String { format!("{}{}", val, self.ty_to_string()) } pub fn bit_width(&self) -> Option { Some(match *self { UintTy::Usize => return None, UintTy::U8 => 8, UintTy::U16 => 16, UintTy::U32 => 32, UintTy::U64 => 64, UintTy::U128 => 128, }) } } impl fmt::Debug for UintTy { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Display::fmt(self, f) } } impl fmt::Display for UintTy { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.ty_to_string()) } } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Copy, PartialOrd, Ord)] pub enum FloatTy { F32, F64, } impl fmt::Debug for FloatTy { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Display::fmt(self, f) } } impl fmt::Display for FloatTy { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.ty_to_string()) } } impl FloatTy { pub fn ty_to_string(&self) -> &'static str { match *self { FloatTy::F32 => "f32", FloatTy::F64 => "f64", } } pub fn bit_width(&self) -> usize { match *self { FloatTy::F32 => 32, FloatTy::F64 => 64, } } } // Bind a type to an associated type: `A=Foo`. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct TypeBinding { pub id: NodeId, pub ident: Ident, pub ty: P, pub span: Span, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] pub struct Ty { pub id: NodeId, pub node: TyKind, pub span: Span, } impl fmt::Debug for Ty { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "type({})", pprust::ty_to_string(self)) } } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct BareFnTy { pub unsafety: Unsafety, pub abi: Abi, pub generic_params: Vec, pub decl: P } /// The different kinds of types recognized by the compiler #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum TyKind { /// A variable-length slice (`[T]`) Slice(P), /// A fixed length array (`[T; n]`) Array(P, P), /// A raw pointer (`*const T` or `*mut T`) Ptr(MutTy), /// A reference (`&'a T` or `&'a mut T`) Rptr(Option, MutTy), /// A bare function (e.g. `fn(usize) -> bool`) BareFn(P), /// The never type (`!`) Never, /// A tuple (`(A, B, C, D,...)`) Tup(Vec> ), /// A path (`module::module::...::Type`), optionally /// "qualified", e.g. ` as SomeTrait>::SomeType`. /// /// Type parameters are stored in the Path itself Path(Option, Path), /// A trait object type `Bound1 + Bound2 + Bound3` /// where `Bound` is a trait or a lifetime. TraitObject(TyParamBounds, TraitObjectSyntax), /// An `impl Bound1 + Bound2 + Bound3` type /// where `Bound` is a trait or a lifetime. ImplTrait(TyParamBounds), /// No-op; kept solely so that we can pretty-print faithfully Paren(P), /// Unused for now Typeof(P), /// TyKind::Infer means the type should be inferred instead of it having been /// specified. This can appear anywhere in a type. Infer, /// Inferred type of a `self` or `&self` argument in a method. ImplicitSelf, // A macro in the type position. Mac(Mac), /// Placeholder for a kind that has failed to be defined. Err, } /// Syntax used to declare a trait object. #[derive(Clone, Copy, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum TraitObjectSyntax { Dyn, None, } /// Inline assembly dialect. /// /// E.g. `"intel"` as in `asm!("mov eax, 2" : "={eax}"(result) : : : "intel")` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum AsmDialect { Att, Intel, } /// Inline assembly. /// /// E.g. `"={eax}"(result)` as in `asm!("mov eax, 2" : "={eax}"(result) : : : "intel")` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct InlineAsmOutput { pub constraint: Symbol, pub expr: P, pub is_rw: bool, pub is_indirect: bool, } /// Inline assembly. /// /// E.g. `asm!("NOP");` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct InlineAsm { pub asm: Symbol, pub asm_str_style: StrStyle, pub outputs: Vec, pub inputs: Vec<(Symbol, P)>, pub clobbers: Vec, pub volatile: bool, pub alignstack: bool, pub dialect: AsmDialect, pub ctxt: SyntaxContext, } /// An argument in a function header. /// /// E.g. `bar: usize` as in `fn foo(bar: usize)` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Arg { pub ty: P, pub pat: P, pub id: NodeId, } /// Alternative representation for `Arg`s describing `self` parameter of methods. /// /// E.g. `&mut self` as in `fn foo(&mut self)` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum SelfKind { /// `self`, `mut self` Value(Mutability), /// `&'lt self`, `&'lt mut self` Region(Option, Mutability), /// `self: TYPE`, `mut self: TYPE` Explicit(P, Mutability), } pub type ExplicitSelf = Spanned; impl Arg { pub fn to_self(&self) -> Option { if let PatKind::Ident(BindingMode::ByValue(mutbl), ident, _) = self.pat.node { if ident.node.name == keywords::SelfValue.name() { return match self.ty.node { TyKind::ImplicitSelf => Some(respan(self.pat.span, SelfKind::Value(mutbl))), TyKind::Rptr(lt, MutTy{ref ty, mutbl}) if ty.node == TyKind::ImplicitSelf => { Some(respan(self.pat.span, SelfKind::Region(lt, mutbl))) } _ => Some(respan(self.pat.span.to(self.ty.span), SelfKind::Explicit(self.ty.clone(), mutbl))), } } } None } pub fn is_self(&self) -> bool { if let PatKind::Ident(_, ident, _) = self.pat.node { ident.node.name == keywords::SelfValue.name() } else { false } } pub fn from_self(eself: ExplicitSelf, eself_ident: SpannedIdent) -> Arg { let span = eself.span.to(eself_ident.span); let infer_ty = P(Ty { id: DUMMY_NODE_ID, node: TyKind::ImplicitSelf, span, }); let arg = |mutbl, ty| Arg { pat: P(Pat { id: DUMMY_NODE_ID, node: PatKind::Ident(BindingMode::ByValue(mutbl), eself_ident, None), span, }), ty, id: DUMMY_NODE_ID, }; match eself.node { SelfKind::Explicit(ty, mutbl) => arg(mutbl, ty), SelfKind::Value(mutbl) => arg(mutbl, infer_ty), SelfKind::Region(lt, mutbl) => arg(Mutability::Immutable, P(Ty { id: DUMMY_NODE_ID, node: TyKind::Rptr(lt, MutTy { ty: infer_ty, mutbl: mutbl }), span, })), } } } /// Header (not the body) of a function declaration. /// /// E.g. `fn foo(bar: baz)` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct FnDecl { pub inputs: Vec, pub output: FunctionRetTy, pub variadic: bool } impl FnDecl { pub fn get_self(&self) -> Option { self.inputs.get(0).and_then(Arg::to_self) } pub fn has_self(&self) -> bool { self.inputs.get(0).map(Arg::is_self).unwrap_or(false) } } /// Is the trait definition an auto trait? #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum IsAuto { Yes, No } #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum Unsafety { Unsafe, Normal, } #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum Constness { Const, NotConst, } #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum Defaultness { Default, Final, } impl fmt::Display for Unsafety { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Display::fmt(match *self { Unsafety::Normal => "normal", Unsafety::Unsafe => "unsafe", }, f) } } #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] pub enum ImplPolarity { /// `impl Trait for Type` Positive, /// `impl !Trait for Type` Negative, } impl fmt::Debug for ImplPolarity { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { ImplPolarity::Positive => "positive".fmt(f), ImplPolarity::Negative => "negative".fmt(f), } } } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum FunctionRetTy { /// Return type is not specified. /// /// Functions default to `()` and /// closures default to inference. Span points to where return /// type would be inserted. Default(Span), /// Everything else Ty(P), } impl FunctionRetTy { pub fn span(&self) -> Span { match *self { FunctionRetTy::Default(span) => span, FunctionRetTy::Ty(ref ty) => ty.span, } } } /// Module declaration. /// /// E.g. `mod foo;` or `mod foo { .. }` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Mod { /// A span from the first token past `{` to the last token until `}`. /// For `mod foo;`, the inner span ranges from the first token /// to the last token in the external file. pub inner: Span, pub items: Vec>, } /// Foreign module declaration. /// /// E.g. `extern { .. }` or `extern C { .. }` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct ForeignMod { pub abi: Abi, pub items: Vec, } /// Global inline assembly /// /// aka module-level assembly or file-scoped assembly #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub struct GlobalAsm { pub asm: Symbol, pub ctxt: SyntaxContext, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct EnumDef { pub variants: Vec, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Variant_ { pub name: Ident, pub attrs: Vec, pub data: VariantData, /// Explicit discriminant, e.g. `Foo = 1` pub disr_expr: Option>, } pub type Variant = Spanned; #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum UseTreeKind { Simple(Ident), Glob, Nested(Vec<(UseTree, NodeId)>), } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct UseTree { pub kind: UseTreeKind, pub prefix: Path, pub span: Span, } /// Distinguishes between Attributes that decorate items and Attributes that /// are contained as statements within items. These two cases need to be /// distinguished for pretty-printing. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub enum AttrStyle { Outer, Inner, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] pub struct AttrId(pub usize); /// Meta-data associated with an item /// Doc-comments are promoted to attributes that have is_sugared_doc = true #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Attribute { pub id: AttrId, pub style: AttrStyle, pub path: Path, pub tokens: TokenStream, pub is_sugared_doc: bool, pub span: Span, } /// TraitRef's appear in impls. /// /// resolve maps each TraitRef's ref_id to its defining trait; that's all /// that the ref_id is for. The impl_id maps to the "self type" of this impl. /// If this impl is an ItemKind::Impl, the impl_id is redundant (it could be the /// same as the impl's node id). #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct TraitRef { pub path: Path, pub ref_id: NodeId, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct PolyTraitRef { /// The `'a` in `<'a> Foo<&'a T>` pub bound_generic_params: Vec, /// The `Foo<&'a T>` in `<'a> Foo<&'a T>` pub trait_ref: TraitRef, pub span: Span, } impl PolyTraitRef { pub fn new(generic_params: Vec, path: Path, span: Span) -> Self { PolyTraitRef { bound_generic_params: generic_params, trait_ref: TraitRef { path: path, ref_id: DUMMY_NODE_ID }, span, } } } #[derive(Copy, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum CrateSugar { /// Source is `pub(crate)` PubCrate, /// Source is (just) `crate` JustCrate, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum Visibility { Public, Crate(Span, CrateSugar), Restricted { path: P, id: NodeId }, Inherited, } /// Field of a struct. /// /// E.g. `bar: usize` as in `struct Foo { bar: usize }` #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct StructField { pub span: Span, pub ident: Option, pub vis: Visibility, pub id: NodeId, pub ty: P, pub attrs: Vec, } /// Fields and Ids of enum variants and structs /// /// For enum variants: `NodeId` represents both an Id of the variant itself (relevant for all /// variant kinds) and an Id of the variant's constructor (not relevant for `Struct`-variants). /// One shared Id can be successfully used for these two purposes. /// Id of the whole enum lives in `Item`. /// /// For structs: `NodeId` represents an Id of the structure's constructor, so it is not actually /// used for `Struct`-structs (but still presents). Structures don't have an analogue of "Id of /// the variant itself" from enum variants. /// Id of the whole struct lives in `Item`. #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum VariantData { /// Struct variant. /// /// E.g. `Bar { .. }` as in `enum Foo { Bar { .. } }` Struct(Vec, NodeId), /// Tuple variant. /// /// E.g. `Bar(..)` as in `enum Foo { Bar(..) }` Tuple(Vec, NodeId), /// Unit variant. /// /// E.g. `Bar = ..` as in `enum Foo { Bar = .. }` Unit(NodeId), } impl VariantData { pub fn fields(&self) -> &[StructField] { match *self { VariantData::Struct(ref fields, _) | VariantData::Tuple(ref fields, _) => fields, _ => &[], } } pub fn id(&self) -> NodeId { match *self { VariantData::Struct(_, id) | VariantData::Tuple(_, id) | VariantData::Unit(id) => id } } pub fn is_struct(&self) -> bool { if let VariantData::Struct(..) = *self { true } else { false } } pub fn is_tuple(&self) -> bool { if let VariantData::Tuple(..) = *self { true } else { false } } pub fn is_unit(&self) -> bool { if let VariantData::Unit(..) = *self { true } else { false } } } /// An item /// /// The name might be a dummy name in case of anonymous items #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct Item { pub ident: Ident, pub attrs: Vec, pub id: NodeId, pub node: ItemKind, pub vis: Visibility, pub span: Span, /// Original tokens this item was parsed from. This isn't necessarily /// available for all items, although over time more and more items should /// have this be `Some`. Right now this is primarily used for procedural /// macros, notably custom attributes. /// /// Note that the tokens here do not include the outer attributes, but will /// include inner attributes. pub tokens: Option, } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum ItemKind { /// An `extern crate` item, with optional original crate name. /// /// E.g. `extern crate foo` or `extern crate foo_bar as foo` ExternCrate(Option), /// A use declaration (`use` or `pub use`) item. /// /// E.g. `use foo;`, `use foo::bar;` or `use foo::bar as FooBar;` Use(P), /// A static item (`static` or `pub static`). /// /// E.g. `static FOO: i32 = 42;` or `static FOO: &'static str = "bar";` Static(P, Mutability, P), /// A constant item (`const` or `pub const`). /// /// E.g. `const FOO: i32 = 42;` Const(P, P), /// A function declaration (`fn` or `pub fn`). /// /// E.g. `fn foo(bar: usize) -> usize { .. }` Fn(P, Unsafety, Spanned, Abi, Generics, P), /// A module declaration (`mod` or `pub mod`). /// /// E.g. `mod foo;` or `mod foo { .. }` Mod(Mod), /// An external module (`extern` or `pub extern`). /// /// E.g. `extern {}` or `extern "C" {}` ForeignMod(ForeignMod), /// Module-level inline assembly (from `global_asm!()`) GlobalAsm(P), /// A type alias (`type` or `pub type`). /// /// E.g. `type Foo = Bar;` Ty(P, Generics), /// An enum definition (`enum` or `pub enum`). /// /// E.g. `enum Foo { C, D }` Enum(EnumDef, Generics), /// A struct definition (`struct` or `pub struct`). /// /// E.g. `struct Foo { x: A }` Struct(VariantData, Generics), /// A union definition (`union` or `pub union`). /// /// E.g. `union Foo { x: A, y: B }` Union(VariantData, Generics), /// A Trait declaration (`trait` or `pub trait`). /// /// E.g. `trait Foo { .. }`, `trait Foo { .. }` or `auto trait Foo {}` Trait(IsAuto, Unsafety, Generics, TyParamBounds, Vec), /// Trait alias /// /// E.g. `trait Foo = Bar + Quux;` TraitAlias(Generics, TyParamBounds), /// An implementation. /// /// E.g. `impl Foo { .. }` or `impl Trait for Foo { .. }` Impl(Unsafety, ImplPolarity, Defaultness, Generics, Option, // (optional) trait this impl implements P, // self Vec), /// A macro invocation. /// /// E.g. `macro_rules! foo { .. }` or `foo!(..)` Mac(Mac), /// A macro definition. MacroDef(MacroDef), } impl ItemKind { pub fn descriptive_variant(&self) -> &str { match *self { ItemKind::ExternCrate(..) => "extern crate", ItemKind::Use(..) => "use", ItemKind::Static(..) => "static item", ItemKind::Const(..) => "constant item", ItemKind::Fn(..) => "function", ItemKind::Mod(..) => "module", ItemKind::ForeignMod(..) => "foreign module", ItemKind::GlobalAsm(..) => "global asm", ItemKind::Ty(..) => "type alias", ItemKind::Enum(..) => "enum", ItemKind::Struct(..) => "struct", ItemKind::Union(..) => "union", ItemKind::Trait(..) => "trait", ItemKind::TraitAlias(..) => "trait alias", ItemKind::Mac(..) | ItemKind::MacroDef(..) | ItemKind::Impl(..) => "item" } } } #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub struct ForeignItem { pub ident: Ident, pub attrs: Vec, pub node: ForeignItemKind, pub id: NodeId, pub span: Span, pub vis: Visibility, } /// An item within an `extern` block #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] pub enum ForeignItemKind { /// A foreign function Fn(P, Generics), /// A foreign static item (`static ext: u8`), with optional mutability /// (the boolean is true when mutable) Static(P, bool), /// A foreign type Ty, } impl ForeignItemKind { pub fn descriptive_variant(&self) -> &str { match *self { ForeignItemKind::Fn(..) => "foreign function", ForeignItemKind::Static(..) => "foreign static item", ForeignItemKind::Ty => "foreign type", } } } #[cfg(test)] mod tests { use serialize; use super::*; // are ASTs encodable? #[test] fn check_asts_encodable() { fn assert_encodable() {} assert_encodable::(); } } ================================================ FILE: examples/rust/keywords.txt ================================================ false true as async await become break continue do else for if in loop match move return try typeof unsafe use while yield 'static Self abstract box const crate dyn enum extern final fn impl let macro mod mut override priv pub ref self static struct super trait type union unsized virtual where ================================================ FILE: examples/rust/scratch.rs ================================================ fn f() { match self.node { Foo::PatKind::Ident(_) => 1 } } ================================================ FILE: examples/typescript/keywords.txt ================================================ abstract arguments class const declare enum export extends from function implements import interface let module namespace package private protected public static super this type var void with await break case catch continue debugger default delete do in of else eval finally for if instanceof new return switch throw try typeof while yield null true false undefined ================================================ FILE: examples/typescript/parser.ts ================================================ /// /// namespace ts { const enum SignatureFlags { None = 0, Yield = 1 << 0, Await = 1 << 1, Type = 1 << 2, RequireCompleteParameterList = 1 << 3, IgnoreMissingOpenBrace = 1 << 4, JSDoc = 1 << 5, } // tslint:disable variable-name let NodeConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let TokenConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let IdentifierConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let SourceFileConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; // tslint:enable variable-name export function createNode(kind: SyntaxKind, pos?: number, end?: number): Node { if (kind === SyntaxKind.SourceFile) { return new (SourceFileConstructor || (SourceFileConstructor = objectAllocator.getSourceFileConstructor()))(kind, pos, end); } else if (kind === SyntaxKind.Identifier) { return new (IdentifierConstructor || (IdentifierConstructor = objectAllocator.getIdentifierConstructor()))(kind, pos, end); } else if (!isNodeKind(kind)) { return new (TokenConstructor || (TokenConstructor = objectAllocator.getTokenConstructor()))(kind, pos, end); } else { return new (NodeConstructor || (NodeConstructor = objectAllocator.getNodeConstructor()))(kind, pos, end); } } function visitNode(cbNode: (node: Node) => T, node: Node): T | undefined { return node && cbNode(node); } function visitNodes(cbNode: (node: Node) => T, cbNodes: (node: NodeArray) => T | undefined, nodes: NodeArray): T | undefined { if (nodes) { if (cbNodes) { return cbNodes(nodes); } for (const node of nodes) { const result = cbNode(node); if (result) { return result; } } } } /** * Invokes a callback for each child of the given node. The 'cbNode' callback is invoked for all child nodes * stored in properties. If a 'cbNodes' callback is specified, it is invoked for embedded arrays; otherwise, * embedded arrays are flattened and the 'cbNode' callback is invoked for each element. If a callback returns * a truthy value, iteration stops and that value is returned. Otherwise, undefined is returned. * * @param node a given node to visit its children * @param cbNode a callback to be invoked for all child nodes * @param cbNodes a callback to be invoked for embedded array * * @remarks `forEachChild` must visit the children of a node in the order * that they appear in the source code. The language service depends on this property to locate nodes by position. */ export function forEachChild(node: Node, cbNode: (node: Node) => T | undefined, cbNodes?: (nodes: NodeArray) => T | undefined): T | undefined { if (!node || node.kind <= SyntaxKind.LastToken) { return; } switch (node.kind) { case SyntaxKind.QualifiedName: return visitNode(cbNode, (node).left) || visitNode(cbNode, (node).right); case SyntaxKind.TypeParameter: return visitNode(cbNode, (node).name) || visitNode(cbNode, (node).constraint) || visitNode(cbNode, (node).default) || visitNode(cbNode, (node).expression); case SyntaxKind.ShorthandPropertyAssignment: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).questionToken) || visitNode(cbNode, (node).equalsToken) || visitNode(cbNode, (node).objectAssignmentInitializer); case SyntaxKind.SpreadAssignment: return visitNode(cbNode, (node).expression); case SyntaxKind.Parameter: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).dotDotDotToken) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).questionToken) || visitNode(cbNode, (node).type) || visitNode(cbNode, (node).initializer); case SyntaxKind.PropertyDeclaration: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).questionToken) || visitNode(cbNode, (node).exclamationToken) || visitNode(cbNode, (node).type) || visitNode(cbNode, (node).initializer); case SyntaxKind.PropertySignature: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).questionToken) || visitNode(cbNode, (node).type) || visitNode(cbNode, (node).initializer); case SyntaxKind.PropertyAssignment: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).questionToken) || visitNode(cbNode, (node).initializer); case SyntaxKind.VariableDeclaration: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).exclamationToken) || visitNode(cbNode, (node).type) || visitNode(cbNode, (node).initializer); case SyntaxKind.BindingElement: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).dotDotDotToken) || visitNode(cbNode, (node).propertyName) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).initializer); case SyntaxKind.FunctionType: case SyntaxKind.ConstructorType: case SyntaxKind.CallSignature: case SyntaxKind.ConstructSignature: case SyntaxKind.IndexSignature: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNodes(cbNode, cbNodes, (node).typeParameters) || visitNodes(cbNode, cbNodes, (node).parameters) || visitNode(cbNode, (node).type); case SyntaxKind.MethodDeclaration: case SyntaxKind.MethodSignature: case SyntaxKind.Constructor: case SyntaxKind.GetAccessor: case SyntaxKind.SetAccessor: case SyntaxKind.FunctionExpression: case SyntaxKind.FunctionDeclaration: case SyntaxKind.ArrowFunction: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).asteriskToken) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).questionToken) || visitNodes(cbNode, cbNodes, (node).typeParameters) || visitNodes(cbNode, cbNodes, (node).parameters) || visitNode(cbNode, (node).type) || visitNode(cbNode, (node).equalsGreaterThanToken) || visitNode(cbNode, (node).body); case SyntaxKind.TypeReference: return visitNode(cbNode, (node).typeName) || visitNodes(cbNode, cbNodes, (node).typeArguments); case SyntaxKind.TypePredicate: return visitNode(cbNode, (node).parameterName) || visitNode(cbNode, (node).type); case SyntaxKind.TypeQuery: return visitNode(cbNode, (node).exprName); case SyntaxKind.TypeLiteral: return visitNodes(cbNode, cbNodes, (node).members); case SyntaxKind.ArrayType: return visitNode(cbNode, (node).elementType); case SyntaxKind.TupleType: return visitNodes(cbNode, cbNodes, (node).elementTypes); case SyntaxKind.UnionType: case SyntaxKind.IntersectionType: return visitNodes(cbNode, cbNodes, (node).types); case SyntaxKind.ConditionalType: return visitNode(cbNode, (node).checkType) || visitNode(cbNode, (node).extendsType) || visitNode(cbNode, (node).trueType) || visitNode(cbNode, (node).falseType); case SyntaxKind.InferType: return visitNode(cbNode, (node).typeParameter); case SyntaxKind.ParenthesizedType: case SyntaxKind.TypeOperator: return visitNode(cbNode, (node).type); case SyntaxKind.IndexedAccessType: return visitNode(cbNode, (node).objectType) || visitNode(cbNode, (node).indexType); case SyntaxKind.MappedType: return visitNode(cbNode, (node).readonlyToken) || visitNode(cbNode, (node).typeParameter) || visitNode(cbNode, (node).questionToken) || visitNode(cbNode, (node).type); case SyntaxKind.LiteralType: return visitNode(cbNode, (node).literal); case SyntaxKind.ObjectBindingPattern: case SyntaxKind.ArrayBindingPattern: return visitNodes(cbNode, cbNodes, (node).elements); case SyntaxKind.ArrayLiteralExpression: return visitNodes(cbNode, cbNodes, (node).elements); case SyntaxKind.ObjectLiteralExpression: return visitNodes(cbNode, cbNodes, (node).properties); case SyntaxKind.PropertyAccessExpression: return visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).name); case SyntaxKind.ElementAccessExpression: return visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).argumentExpression); case SyntaxKind.CallExpression: case SyntaxKind.NewExpression: return visitNode(cbNode, (node).expression) || visitNodes(cbNode, cbNodes, (node).typeArguments) || visitNodes(cbNode, cbNodes, (node).arguments); case SyntaxKind.TaggedTemplateExpression: return visitNode(cbNode, (node).tag) || visitNode(cbNode, (node).template); case SyntaxKind.TypeAssertionExpression: return visitNode(cbNode, (node).type) || visitNode(cbNode, (node).expression); case SyntaxKind.ParenthesizedExpression: return visitNode(cbNode, (node).expression); case SyntaxKind.DeleteExpression: return visitNode(cbNode, (node).expression); case SyntaxKind.TypeOfExpression: return visitNode(cbNode, (node).expression); case SyntaxKind.VoidExpression: return visitNode(cbNode, (node).expression); case SyntaxKind.PrefixUnaryExpression: return visitNode(cbNode, (node).operand); case SyntaxKind.YieldExpression: return visitNode(cbNode, (node).asteriskToken) || visitNode(cbNode, (node).expression); case SyntaxKind.AwaitExpression: return visitNode(cbNode, (node).expression); case SyntaxKind.PostfixUnaryExpression: return visitNode(cbNode, (node).operand); case SyntaxKind.BinaryExpression: return visitNode(cbNode, (node).left) || visitNode(cbNode, (node).operatorToken) || visitNode(cbNode, (node).right); case SyntaxKind.AsExpression: return visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).type); case SyntaxKind.NonNullExpression: return visitNode(cbNode, (node).expression); case SyntaxKind.MetaProperty: return visitNode(cbNode, (node).name); case SyntaxKind.ConditionalExpression: return visitNode(cbNode, (node).condition) || visitNode(cbNode, (node).questionToken) || visitNode(cbNode, (node).whenTrue) || visitNode(cbNode, (node).colonToken) || visitNode(cbNode, (node).whenFalse); case SyntaxKind.SpreadElement: return visitNode(cbNode, (node).expression); case SyntaxKind.Block: case SyntaxKind.ModuleBlock: return visitNodes(cbNode, cbNodes, (node).statements); case SyntaxKind.SourceFile: return visitNodes(cbNode, cbNodes, (node).statements) || visitNode(cbNode, (node).endOfFileToken); case SyntaxKind.VariableStatement: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).declarationList); case SyntaxKind.VariableDeclarationList: return visitNodes(cbNode, cbNodes, (node).declarations); case SyntaxKind.ExpressionStatement: return visitNode(cbNode, (node).expression); case SyntaxKind.IfStatement: return visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).thenStatement) || visitNode(cbNode, (node).elseStatement); case SyntaxKind.DoStatement: return visitNode(cbNode, (node).statement) || visitNode(cbNode, (node).expression); case SyntaxKind.WhileStatement: return visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).statement); case SyntaxKind.ForStatement: return visitNode(cbNode, (node).initializer) || visitNode(cbNode, (node).condition) || visitNode(cbNode, (node).incrementor) || visitNode(cbNode, (node).statement); case SyntaxKind.ForInStatement: return visitNode(cbNode, (node).initializer) || visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).statement); case SyntaxKind.ForOfStatement: return visitNode(cbNode, (node).awaitModifier) || visitNode(cbNode, (node).initializer) || visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).statement); case SyntaxKind.ContinueStatement: case SyntaxKind.BreakStatement: return visitNode(cbNode, (node).label); case SyntaxKind.ReturnStatement: return visitNode(cbNode, (node).expression); case SyntaxKind.WithStatement: return visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).statement); case SyntaxKind.SwitchStatement: return visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).caseBlock); case SyntaxKind.CaseBlock: return visitNodes(cbNode, cbNodes, (node).clauses); case SyntaxKind.CaseClause: return visitNode(cbNode, (node).expression) || visitNodes(cbNode, cbNodes, (node).statements); case SyntaxKind.DefaultClause: return visitNodes(cbNode, cbNodes, (node).statements); case SyntaxKind.LabeledStatement: return visitNode(cbNode, (node).label) || visitNode(cbNode, (node).statement); case SyntaxKind.ThrowStatement: return visitNode(cbNode, (node).expression); case SyntaxKind.TryStatement: return visitNode(cbNode, (node).tryBlock) || visitNode(cbNode, (node).catchClause) || visitNode(cbNode, (node).finallyBlock); case SyntaxKind.CatchClause: return visitNode(cbNode, (node).variableDeclaration) || visitNode(cbNode, (node).block); case SyntaxKind.Decorator: return visitNode(cbNode, (node).expression); case SyntaxKind.ClassDeclaration: case SyntaxKind.ClassExpression: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNodes(cbNode, cbNodes, (node).typeParameters) || visitNodes(cbNode, cbNodes, (node).heritageClauses) || visitNodes(cbNode, cbNodes, (node).members); case SyntaxKind.InterfaceDeclaration: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNodes(cbNode, cbNodes, (node).typeParameters) || visitNodes(cbNode, cbNodes, (node).heritageClauses) || visitNodes(cbNode, cbNodes, (node).members); case SyntaxKind.TypeAliasDeclaration: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNodes(cbNode, cbNodes, (node).typeParameters) || visitNode(cbNode, (node).type); case SyntaxKind.EnumDeclaration: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNodes(cbNode, cbNodes, (node).members); case SyntaxKind.EnumMember: return visitNode(cbNode, (node).name) || visitNode(cbNode, (node).initializer); case SyntaxKind.ModuleDeclaration: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).body); case SyntaxKind.ImportEqualsDeclaration: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).name) || visitNode(cbNode, (node).moduleReference); case SyntaxKind.ImportDeclaration: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).importClause) || visitNode(cbNode, (node).moduleSpecifier); case SyntaxKind.ImportClause: return visitNode(cbNode, (node).name) || visitNode(cbNode, (node).namedBindings); case SyntaxKind.NamespaceExportDeclaration: return visitNode(cbNode, (node).name); case SyntaxKind.NamespaceImport: return visitNode(cbNode, (node).name); case SyntaxKind.NamedImports: case SyntaxKind.NamedExports: return visitNodes(cbNode, cbNodes, (node).elements); case SyntaxKind.ExportDeclaration: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).exportClause) || visitNode(cbNode, (node).moduleSpecifier); case SyntaxKind.ImportSpecifier: case SyntaxKind.ExportSpecifier: return visitNode(cbNode, (node).propertyName) || visitNode(cbNode, (node).name); case SyntaxKind.ExportAssignment: return visitNodes(cbNode, cbNodes, node.decorators) || visitNodes(cbNode, cbNodes, node.modifiers) || visitNode(cbNode, (node).expression); case SyntaxKind.TemplateExpression: return visitNode(cbNode, (node).head) || visitNodes(cbNode, cbNodes, (node).templateSpans); case SyntaxKind.TemplateSpan: return visitNode(cbNode, (node).expression) || visitNode(cbNode, (node).literal); case SyntaxKind.ComputedPropertyName: return visitNode(cbNode, (node).expression); case SyntaxKind.HeritageClause: return visitNodes(cbNode, cbNodes, (node).types); case SyntaxKind.ExpressionWithTypeArguments: return visitNode(cbNode, (node).expression) || visitNodes(cbNode, cbNodes, (node).typeArguments); case SyntaxKind.ExternalModuleReference: return visitNode(cbNode, (node).expression); case SyntaxKind.MissingDeclaration: return visitNodes(cbNode, cbNodes, node.decorators); case SyntaxKind.CommaListExpression: return visitNodes(cbNode, cbNodes, (node).elements); case SyntaxKind.JsxElement: return visitNode(cbNode, (node).openingElement) || visitNodes(cbNode, cbNodes, (node).children) || visitNode(cbNode, (node).closingElement); case SyntaxKind.JsxFragment: return visitNode(cbNode, (node).openingFragment) || visitNodes(cbNode, cbNodes, (node).children) || visitNode(cbNode, (node).closingFragment); case SyntaxKind.JsxSelfClosingElement: case SyntaxKind.JsxOpeningElement: return visitNode(cbNode, (node).tagName) || visitNode(cbNode, (node).attributes); case SyntaxKind.JsxAttributes: return visitNodes(cbNode, cbNodes, (node).properties); case SyntaxKind.JsxAttribute: return visitNode(cbNode, (node).name) || visitNode(cbNode, (node).initializer); case SyntaxKind.JsxSpreadAttribute: return visitNode(cbNode, (node).expression); case SyntaxKind.JsxExpression: return visitNode(cbNode, (node as JsxExpression).dotDotDotToken) || visitNode(cbNode, (node as JsxExpression).expression); case SyntaxKind.JsxClosingElement: return visitNode(cbNode, (node).tagName); case SyntaxKind.JSDocTypeExpression: return visitNode(cbNode, (node).type); case SyntaxKind.JSDocNonNullableType: return visitNode(cbNode, (node).type); case SyntaxKind.JSDocNullableType: return visitNode(cbNode, (node).type); case SyntaxKind.JSDocOptionalType: return visitNode(cbNode, (node).type); case SyntaxKind.JSDocFunctionType: return visitNodes(cbNode, cbNodes, (node).parameters) || visitNode(cbNode, (node).type); case SyntaxKind.JSDocVariadicType: return visitNode(cbNode, (node).type); case SyntaxKind.JSDocComment: return visitNodes(cbNode, cbNodes, (node).tags); case SyntaxKind.JSDocParameterTag: case SyntaxKind.JSDocPropertyTag: if ((node as JSDocPropertyLikeTag).isNameFirst) { return visitNode(cbNode, (node).name) || visitNode(cbNode, (node).typeExpression); } else { return visitNode(cbNode, (node).typeExpression) || visitNode(cbNode, (node).name); } case SyntaxKind.JSDocReturnTag: return visitNode(cbNode, (node).typeExpression); case SyntaxKind.JSDocTypeTag: return visitNode(cbNode, (node).typeExpression); case SyntaxKind.JSDocAugmentsTag: return visitNode(cbNode, (node).class); case SyntaxKind.JSDocTemplateTag: return visitNodes(cbNode, cbNodes, (node).typeParameters); case SyntaxKind.JSDocTypedefTag: if ((node as JSDocTypedefTag).typeExpression && (node as JSDocTypedefTag).typeExpression.kind === SyntaxKind.JSDocTypeExpression) { return visitNode(cbNode, (node).typeExpression) || visitNode(cbNode, (node).fullName); } else { return visitNode(cbNode, (node).fullName) || visitNode(cbNode, (node).typeExpression); } case SyntaxKind.JSDocTypeLiteral: if ((node as JSDocTypeLiteral).jsDocPropertyTags) { for (const tag of (node as JSDocTypeLiteral).jsDocPropertyTags) { visitNode(cbNode, tag); } } return; case SyntaxKind.PartiallyEmittedExpression: return visitNode(cbNode, (node).expression); } } export function createSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, setParentNodes = false, scriptKind?: ScriptKind): SourceFile { performance.mark("beforeParse"); const result = Parser.parseSourceFile(fileName, sourceText, languageVersion, /*syntaxCursor*/ undefined, setParentNodes, scriptKind); performance.mark("afterParse"); performance.measure("Parse", "beforeParse", "afterParse"); return result; } export function parseIsolatedEntityName(text: string, languageVersion: ScriptTarget): EntityName { return Parser.parseIsolatedEntityName(text, languageVersion); } /** * Parse json text into SyntaxTree and return node and parse errors if any * @param fileName * @param sourceText */ export function parseJsonText(fileName: string, sourceText: string): JsonSourceFile { return Parser.parseJsonText(fileName, sourceText); } // See also `isExternalOrCommonJsModule` in utilities.ts export function isExternalModule(file: SourceFile): boolean { return file.externalModuleIndicator !== undefined; } // Produces a new SourceFile for the 'newText' provided. The 'textChangeRange' parameter // indicates what changed between the 'text' that this SourceFile has and the 'newText'. // The SourceFile will be created with the compiler attempting to reuse as many nodes from // this file as possible. // // Note: this function mutates nodes from this SourceFile. That means any existing nodes // from this SourceFile that are being held onto may change as a result (including // becoming detached from any SourceFile). It is recommended that this SourceFile not // be used once 'update' is called on it. export function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile { const newSourceFile = IncrementalParser.updateSourceFile(sourceFile, newText, textChangeRange, aggressiveChecks); // Because new source file node is created, it may not have the flag PossiblyContainDynamicImport. This is the case if there is no new edit to add dynamic import. // We will manually port the flag to the new source file. newSourceFile.flags |= (sourceFile.flags & NodeFlags.PossiblyContainsDynamicImport); return newSourceFile; } /* @internal */ export function parseIsolatedJSDocComment(content: string, start?: number, length?: number) { const result = Parser.JSDocParser.parseIsolatedJSDocComment(content, start, length); if (result && result.jsDoc) { // because the jsDocComment was parsed out of the source file, it might // not be covered by the fixupParentReferences. Parser.fixupParentReferences(result.jsDoc); } return result; } /* @internal */ // Exposed only for testing. export function parseJSDocTypeExpressionForTests(content: string, start?: number, length?: number) { return Parser.JSDocParser.parseJSDocTypeExpressionForTests(content, start, length); } // Implement the parser as a singleton module. We do this for perf reasons because creating // parser instances can actually be expensive enough to impact us on projects with many source // files. namespace Parser { // Share a single scanner across all calls to parse a source file. This helps speed things // up by avoiding the cost of creating/compiling scanners over and over again. const scanner = createScanner(ScriptTarget.Latest, /*skipTrivia*/ true); const disallowInAndDecoratorContext = NodeFlags.DisallowInContext | NodeFlags.DecoratorContext; // capture constructors in 'initializeState' to avoid null checks // tslint:disable variable-name let NodeConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let TokenConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let IdentifierConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let SourceFileConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; // tslint:enable variable-name let sourceFile: SourceFile; let parseDiagnostics: Diagnostic[]; let syntaxCursor: IncrementalParser.SyntaxCursor; let currentToken: SyntaxKind; let sourceText: string; let nodeCount: number; let identifiers: Map; let identifierCount: number; let parsingContext: ParsingContext; // Flags that dictate what parsing context we're in. For example: // Whether or not we are in strict parsing mode. All that changes in strict parsing mode is // that some tokens that would be considered identifiers may be considered keywords. // // When adding more parser context flags, consider which is the more common case that the // flag will be in. This should be the 'false' state for that flag. The reason for this is // that we don't store data in our nodes unless the value is in the *non-default* state. So, // for example, more often than code 'allows-in' (or doesn't 'disallow-in'). We opt for // 'disallow-in' set to 'false'. Otherwise, if we had 'allowsIn' set to 'true', then almost // all nodes would need extra state on them to store this info. // // Note: 'allowIn' and 'allowYield' track 1:1 with the [in] and [yield] concepts in the ES6 // grammar specification. // // An important thing about these context concepts. By default they are effectively inherited // while parsing through every grammar production. i.e. if you don't change them, then when // you parse a sub-production, it will have the same context values as the parent production. // This is great most of the time. After all, consider all the 'expression' grammar productions // and how nearly all of them pass along the 'in' and 'yield' context values: // // EqualityExpression[In, Yield] : // RelationalExpression[?In, ?Yield] // EqualityExpression[?In, ?Yield] == RelationalExpression[?In, ?Yield] // EqualityExpression[?In, ?Yield] != RelationalExpression[?In, ?Yield] // EqualityExpression[?In, ?Yield] === RelationalExpression[?In, ?Yield] // EqualityExpression[?In, ?Yield] !== RelationalExpression[?In, ?Yield] // // Where you have to be careful is then understanding what the points are in the grammar // where the values are *not* passed along. For example: // // SingleNameBinding[Yield,GeneratorParameter] // [+GeneratorParameter]BindingIdentifier[Yield] Initializer[In]opt // [~GeneratorParameter]BindingIdentifier[?Yield]Initializer[In, ?Yield]opt // // Here this is saying that if the GeneratorParameter context flag is set, that we should // explicitly set the 'yield' context flag to false before calling into the BindingIdentifier // and we should explicitly unset the 'yield' context flag before calling into the Initializer. // production. Conversely, if the GeneratorParameter context flag is not set, then we // should leave the 'yield' context flag alone. // // Getting this all correct is tricky and requires careful reading of the grammar to // understand when these values should be changed versus when they should be inherited. // // Note: it should not be necessary to save/restore these flags during speculative/lookahead // parsing. These context flags are naturally stored and restored through normal recursive // descent parsing and unwinding. let contextFlags: NodeFlags; // Whether or not we've had a parse error since creating the last AST node. If we have // encountered an error, it will be stored on the next AST node we create. Parse errors // can be broken down into three categories: // // 1) An error that occurred during scanning. For example, an unterminated literal, or a // character that was completely not understood. // // 2) A token was expected, but was not present. This type of error is commonly produced // by the 'parseExpected' function. // // 3) A token was present that no parsing function was able to consume. This type of error // only occurs in the 'abortParsingListOrMoveToNextToken' function when the parser // decides to skip the token. // // In all of these cases, we want to mark the next node as having had an error before it. // With this mark, we can know in incremental settings if this node can be reused, or if // we have to reparse it. If we don't keep this information around, we may just reuse the // node. in that event we would then not produce the same errors as we did before, causing // significant confusion problems. // // Note: it is necessary that this value be saved/restored during speculative/lookahead // parsing. During lookahead parsing, we will often create a node. That node will have // this value attached, and then this value will be set back to 'false'. If we decide to // rewind, we must get back to the same value we had prior to the lookahead. // // Note: any errors at the end of the file that do not precede a regular node, should get // attached to the EOF token. let parseErrorBeforeNextFinishedNode = false; export function parseSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, syntaxCursor: IncrementalParser.SyntaxCursor, setParentNodes?: boolean, scriptKind?: ScriptKind): SourceFile { scriptKind = ensureScriptKind(fileName, scriptKind); initializeState(sourceText, languageVersion, syntaxCursor, scriptKind); const result = parseSourceFileWorker(fileName, languageVersion, setParentNodes, scriptKind); clearState(); return result; } export function parseIsolatedEntityName(content: string, languageVersion: ScriptTarget): EntityName { // Choice of `isDeclarationFile` should be arbitrary initializeState(content, languageVersion, /*syntaxCursor*/ undefined, ScriptKind.JS); // Prime the scanner. nextToken(); const entityName = parseEntityName(/*allowReservedWords*/ true); const isInvalid = token() === SyntaxKind.EndOfFileToken && !parseDiagnostics.length; clearState(); return isInvalid ? entityName : undefined; } export function parseJsonText(fileName: string, sourceText: string): JsonSourceFile { initializeState(sourceText, ScriptTarget.ES2015, /*syntaxCursor*/ undefined, ScriptKind.JSON); // Set source file so that errors will be reported with this file name sourceFile = createSourceFile(fileName, ScriptTarget.ES2015, ScriptKind.JSON, /*isDeclaration*/ false); const result = sourceFile; // Prime the scanner. nextToken(); if (token() === SyntaxKind.EndOfFileToken) { sourceFile.endOfFileToken = parseTokenNode(); } else if (token() === SyntaxKind.OpenBraceToken || lookAhead(() => token() === SyntaxKind.StringLiteral)) { result.jsonObject = parseObjectLiteralExpression(); sourceFile.endOfFileToken = parseExpectedToken(SyntaxKind.EndOfFileToken, Diagnostics.Unexpected_token); } else { parseExpected(SyntaxKind.OpenBraceToken); } sourceFile.parseDiagnostics = parseDiagnostics; clearState(); return result; } function getLanguageVariant(scriptKind: ScriptKind) { // .tsx and .jsx files are treated as jsx language variant. return scriptKind === ScriptKind.TSX || scriptKind === ScriptKind.JSX || scriptKind === ScriptKind.JS || scriptKind === ScriptKind.JSON ? LanguageVariant.JSX : LanguageVariant.Standard; } function initializeState(_sourceText: string, languageVersion: ScriptTarget, _syntaxCursor: IncrementalParser.SyntaxCursor, scriptKind: ScriptKind) { NodeConstructor = objectAllocator.getNodeConstructor(); TokenConstructor = objectAllocator.getTokenConstructor(); IdentifierConstructor = objectAllocator.getIdentifierConstructor(); SourceFileConstructor = objectAllocator.getSourceFileConstructor(); sourceText = _sourceText; syntaxCursor = _syntaxCursor; parseDiagnostics = []; parsingContext = 0; identifiers = createMap(); identifierCount = 0; nodeCount = 0; switch (scriptKind) { case ScriptKind.JS: case ScriptKind.JSX: case ScriptKind.JSON: contextFlags = NodeFlags.JavaScriptFile; break; default: contextFlags = NodeFlags.None; break; } parseErrorBeforeNextFinishedNode = false; // Initialize and prime the scanner before parsing the source elements. scanner.setText(sourceText); scanner.setOnError(scanError); scanner.setScriptTarget(languageVersion); scanner.setLanguageVariant(getLanguageVariant(scriptKind)); } function clearState() { // Clear out the text the scanner is pointing at, so it doesn't keep anything alive unnecessarily. scanner.setText(""); scanner.setOnError(undefined); // Clear any data. We don't want to accidentally hold onto it for too long. parseDiagnostics = undefined; sourceFile = undefined; identifiers = undefined; syntaxCursor = undefined; sourceText = undefined; } function parseSourceFileWorker(fileName: string, languageVersion: ScriptTarget, setParentNodes: boolean, scriptKind: ScriptKind): SourceFile { const isDeclarationFile = isDeclarationFileName(fileName); if (isDeclarationFile) { contextFlags |= NodeFlags.Ambient; } sourceFile = createSourceFile(fileName, languageVersion, scriptKind, isDeclarationFile); sourceFile.flags = contextFlags; // Prime the scanner. nextToken(); processReferenceComments(sourceFile); sourceFile.statements = parseList(ParsingContext.SourceElements, parseStatement); Debug.assert(token() === SyntaxKind.EndOfFileToken); sourceFile.endOfFileToken = addJSDocComment(parseTokenNode() as EndOfFileToken); setExternalModuleIndicator(sourceFile); sourceFile.nodeCount = nodeCount; sourceFile.identifierCount = identifierCount; sourceFile.identifiers = identifiers; sourceFile.parseDiagnostics = parseDiagnostics; if (setParentNodes) { fixupParentReferences(sourceFile); } return sourceFile; } function addJSDocComment(node: T): T { const comments = getJSDocCommentRanges(node, sourceFile.text); if (comments) { for (const comment of comments) { node.jsDoc = append(node.jsDoc, JSDocParser.parseJSDocComment(node, comment.pos, comment.end - comment.pos)); } } return node; } export function fixupParentReferences(rootNode: Node) { // normally parent references are set during binding. However, for clients that only need // a syntax tree, and no semantic features, then the binding process is an unnecessary // overhead. This functions allows us to set all the parents, without all the expense of // binding. let parent: Node = rootNode; forEachChild(rootNode, visitNode); return; function visitNode(n: Node): void { // walk down setting parents that differ from the parent we think it should be. This // allows us to quickly bail out of setting parents for subtrees during incremental // parsing if (n.parent !== parent) { n.parent = parent; const saveParent = parent; parent = n; forEachChild(n, visitNode); if (hasJSDocNodes(n)) { for (const jsDoc of n.jsDoc) { jsDoc.parent = n; parent = jsDoc; forEachChild(jsDoc, visitNode); } } parent = saveParent; } } } function createSourceFile(fileName: string, languageVersion: ScriptTarget, scriptKind: ScriptKind, isDeclarationFile: boolean): SourceFile { // code from createNode is inlined here so createNode won't have to deal with special case of creating source files // this is quite rare comparing to other nodes and createNode should be as fast as possible const sourceFile = new SourceFileConstructor(SyntaxKind.SourceFile, /*pos*/ 0, /* end */ sourceText.length); nodeCount++; sourceFile.text = sourceText; sourceFile.bindDiagnostics = []; sourceFile.languageVersion = languageVersion; sourceFile.fileName = normalizePath(fileName); sourceFile.languageVariant = getLanguageVariant(scriptKind); sourceFile.isDeclarationFile = isDeclarationFile; sourceFile.scriptKind = scriptKind; return sourceFile; } function setContextFlag(val: boolean, flag: NodeFlags) { if (val) { contextFlags |= flag; } else { contextFlags &= ~flag; } } function setDisallowInContext(val: boolean) { setContextFlag(val, NodeFlags.DisallowInContext); } function setYieldContext(val: boolean) { setContextFlag(val, NodeFlags.YieldContext); } function setDecoratorContext(val: boolean) { setContextFlag(val, NodeFlags.DecoratorContext); } function setAwaitContext(val: boolean) { setContextFlag(val, NodeFlags.AwaitContext); } function doOutsideOfContext(context: NodeFlags, func: () => T): T { // contextFlagsToClear will contain only the context flags that are // currently set that we need to temporarily clear // We don't just blindly reset to the previous flags to ensure // that we do not mutate cached flags for the incremental // parser (ThisNodeHasError, ThisNodeOrAnySubNodesHasError, and // HasAggregatedChildData). const contextFlagsToClear = context & contextFlags; if (contextFlagsToClear) { // clear the requested context flags setContextFlag(/*val*/ false, contextFlagsToClear); const result = func(); // restore the context flags we just cleared setContextFlag(/*val*/ true, contextFlagsToClear); return result; } // no need to do anything special as we are not in any of the requested contexts return func(); } function doInsideOfContext(context: NodeFlags, func: () => T): T { // contextFlagsToSet will contain only the context flags that // are not currently set that we need to temporarily enable. // We don't just blindly reset to the previous flags to ensure // that we do not mutate cached flags for the incremental // parser (ThisNodeHasError, ThisNodeOrAnySubNodesHasError, and // HasAggregatedChildData). const contextFlagsToSet = context & ~contextFlags; if (contextFlagsToSet) { // set the requested context flags setContextFlag(/*val*/ true, contextFlagsToSet); const result = func(); // reset the context flags we just set setContextFlag(/*val*/ false, contextFlagsToSet); return result; } // no need to do anything special as we are already in all of the requested contexts return func(); } function allowInAnd(func: () => T): T { return doOutsideOfContext(NodeFlags.DisallowInContext, func); } function disallowInAnd(func: () => T): T { return doInsideOfContext(NodeFlags.DisallowInContext, func); } function doInYieldContext(func: () => T): T { return doInsideOfContext(NodeFlags.YieldContext, func); } function doInDecoratorContext(func: () => T): T { return doInsideOfContext(NodeFlags.DecoratorContext, func); } function doInAwaitContext(func: () => T): T { return doInsideOfContext(NodeFlags.AwaitContext, func); } function doOutsideOfAwaitContext(func: () => T): T { return doOutsideOfContext(NodeFlags.AwaitContext, func); } function doInYieldAndAwaitContext(func: () => T): T { return doInsideOfContext(NodeFlags.YieldContext | NodeFlags.AwaitContext, func); } function inContext(flags: NodeFlags) { return (contextFlags & flags) !== 0; } function inYieldContext() { return inContext(NodeFlags.YieldContext); } function inDisallowInContext() { return inContext(NodeFlags.DisallowInContext); } function inDecoratorContext() { return inContext(NodeFlags.DecoratorContext); } function inAwaitContext() { return inContext(NodeFlags.AwaitContext); } function parseErrorAtCurrentToken(message: DiagnosticMessage, arg0?: any): void { const start = scanner.getTokenPos(); const length = scanner.getTextPos() - start; parseErrorAtPosition(start, length, message, arg0); } function parseErrorAtPosition(start: number, length: number, message: DiagnosticMessage, arg0?: any): void { // Don't report another error if it would just be at the same position as the last error. const lastError = lastOrUndefined(parseDiagnostics); if (!lastError || start !== lastError.start) { parseDiagnostics.push(createFileDiagnostic(sourceFile, start, length, message, arg0)); } // Mark that we've encountered an error. We'll set an appropriate bit on the next // node we finish so that it can't be reused incrementally. parseErrorBeforeNextFinishedNode = true; } function scanError(message: DiagnosticMessage, length?: number) { const pos = scanner.getTextPos(); parseErrorAtPosition(pos, length || 0, message); } function getNodePos(): number { return scanner.getStartPos(); } // Use this function to access the current token instead of reading the currentToken // variable. Since function results aren't narrowed in control flow analysis, this ensures // that the type checker doesn't make wrong assumptions about the type of the current // token (e.g. a call to nextToken() changes the current token but the checker doesn't // reason about this side effect). Mainstream VMs inline simple functions like this, so // there is no performance penalty. function token(): SyntaxKind { return currentToken; } function nextToken(): SyntaxKind { return currentToken = scanner.scan(); } function reScanGreaterToken(): SyntaxKind { return currentToken = scanner.reScanGreaterToken(); } function reScanSlashToken(): SyntaxKind { return currentToken = scanner.reScanSlashToken(); } function reScanTemplateToken(): SyntaxKind { return currentToken = scanner.reScanTemplateToken(); } function scanJsxIdentifier(): SyntaxKind { return currentToken = scanner.scanJsxIdentifier(); } function scanJsxText(): SyntaxKind { return currentToken = scanner.scanJsxToken(); } function scanJsxAttributeValue(): SyntaxKind { return currentToken = scanner.scanJsxAttributeValue(); } function speculationHelper(callback: () => T, isLookAhead: boolean): T { // Keep track of the state we'll need to rollback to if lookahead fails (or if the // caller asked us to always reset our state). const saveToken = currentToken; const saveParseDiagnosticsLength = parseDiagnostics.length; const saveParseErrorBeforeNextFinishedNode = parseErrorBeforeNextFinishedNode; // Note: it is not actually necessary to save/restore the context flags here. That's // because the saving/restoring of these flags happens naturally through the recursive // descent nature of our parser. However, we still store this here just so we can // assert that invariant holds. const saveContextFlags = contextFlags; // If we're only looking ahead, then tell the scanner to only lookahead as well. // Otherwise, if we're actually speculatively parsing, then tell the scanner to do the // same. const result = isLookAhead ? scanner.lookAhead(callback) : scanner.tryScan(callback); Debug.assert(saveContextFlags === contextFlags); // If our callback returned something 'falsy' or we're just looking ahead, // then unconditionally restore us to where we were. if (!result || isLookAhead) { currentToken = saveToken; parseDiagnostics.length = saveParseDiagnosticsLength; parseErrorBeforeNextFinishedNode = saveParseErrorBeforeNextFinishedNode; } return result; } /** Invokes the provided callback then unconditionally restores the parser to the state it * was in immediately prior to invoking the callback. The result of invoking the callback * is returned from this function. */ function lookAhead(callback: () => T): T { return speculationHelper(callback, /*isLookAhead*/ true); } /** Invokes the provided callback. If the callback returns something falsy, then it restores * the parser to the state it was in immediately prior to invoking the callback. If the * callback returns something truthy, then the parser state is not rolled back. The result * of invoking the callback is returned from this function. */ function tryParse(callback: () => T): T { return speculationHelper(callback, /*isLookAhead*/ false); } // Ignore strict mode flag because we will report an error in type checker instead. function isIdentifier(): boolean { if (token() === SyntaxKind.Identifier) { return true; } // If we have a 'yield' keyword, and we're in the [yield] context, then 'yield' is // considered a keyword and is not an identifier. if (token() === SyntaxKind.YieldKeyword && inYieldContext()) { return false; } // If we have a 'await' keyword, and we're in the [Await] context, then 'await' is // considered a keyword and is not an identifier. if (token() === SyntaxKind.AwaitKeyword && inAwaitContext()) { return false; } return token() > SyntaxKind.LastReservedWord; } function parseExpected(kind: SyntaxKind, diagnosticMessage?: DiagnosticMessage, shouldAdvance = true): boolean { if (token() === kind) { if (shouldAdvance) { nextToken(); } return true; } // Report specific message if provided with one. Otherwise, report generic fallback message. if (diagnosticMessage) { parseErrorAtCurrentToken(diagnosticMessage); } else { parseErrorAtCurrentToken(Diagnostics._0_expected, tokenToString(kind)); } return false; } function parseOptional(t: SyntaxKind): boolean { if (token() === t) { nextToken(); return true; } return false; } function parseOptionalToken(t: TKind): Token; function parseOptionalToken(t: SyntaxKind): Node { if (token() === t) { return parseTokenNode(); } return undefined; } function parseExpectedToken(t: TKind, diagnosticMessage?: DiagnosticMessage, arg0?: any): Token; function parseExpectedToken(t: SyntaxKind, diagnosticMessage?: DiagnosticMessage, arg0?: any): Node { return parseOptionalToken(t) || createMissingNode(t, /*reportAtCurrentPosition*/ false, diagnosticMessage || Diagnostics._0_expected, arg0 || tokenToString(t)); } function parseTokenNode(): T { const node = createNode(token()); nextToken(); return finishNode(node); } function canParseSemicolon() { // If there's a real semicolon, then we can always parse it out. if (token() === SyntaxKind.SemicolonToken) { return true; } // We can parse out an optional semicolon in ASI cases in the following cases. return token() === SyntaxKind.CloseBraceToken || token() === SyntaxKind.EndOfFileToken || scanner.hasPrecedingLineBreak(); } function parseSemicolon(): boolean { if (canParseSemicolon()) { if (token() === SyntaxKind.SemicolonToken) { // consume the semicolon if it was explicitly provided. nextToken(); } return true; } else { return parseExpected(SyntaxKind.SemicolonToken); } } function createNode(kind: SyntaxKind, pos?: number): Node { nodeCount++; const p = pos >= 0 ? pos : scanner.getStartPos(); return isNodeKind(kind) || kind === SyntaxKind.Unknown ? new NodeConstructor(kind, p, p) : kind === SyntaxKind.Identifier ? new IdentifierConstructor(kind, p, p) : new TokenConstructor(kind, p, p); } function createNodeWithJSDoc(kind: SyntaxKind): Node { const node = createNode(kind); if (scanner.getTokenFlags() & TokenFlags.PrecedingJSDocComment) { addJSDocComment(node); } return node; } function createNodeArray(elements: T[], pos: number, end?: number): NodeArray { // Since the element list of a node array is typically created by starting with an empty array and // repeatedly calling push(), the list may not have the optimal memory layout. We invoke slice() for // small arrays (1 to 4 elements) to give the VM a chance to allocate an optimal representation. const length = elements.length; const array = >(length >= 1 && length <= 4 ? elements.slice() : elements); array.pos = pos; array.end = end === undefined ? scanner.getStartPos() : end; return array; } function finishNode(node: T, end?: number): T { node.end = end === undefined ? scanner.getStartPos() : end; if (contextFlags) { node.flags |= contextFlags; } // Keep track on the node if we encountered an error while parsing it. If we did, then // we cannot reuse the node incrementally. Once we've marked this node, clear out the // flag so that we don't mark any subsequent nodes. if (parseErrorBeforeNextFinishedNode) { parseErrorBeforeNextFinishedNode = false; node.flags |= NodeFlags.ThisNodeHasError; } return node; } function createMissingNode(kind: T["kind"], reportAtCurrentPosition: boolean, diagnosticMessage: DiagnosticMessage, arg0?: any): T { if (reportAtCurrentPosition) { parseErrorAtPosition(scanner.getStartPos(), 0, diagnosticMessage, arg0); } else { parseErrorAtCurrentToken(diagnosticMessage, arg0); } const result = createNode(kind); if (kind === SyntaxKind.Identifier) { (result as Identifier).escapedText = "" as __String; } else if (isLiteralKind(kind) || isTemplateLiteralKind(kind)) { (result as LiteralLikeNode).text = ""; } return finishNode(result) as T; } function internIdentifier(text: string): string { let identifier = identifiers.get(text); if (identifier === undefined) { identifiers.set(text, identifier = text); } return identifier; } // An identifier that starts with two underscores has an extra underscore character prepended to it to avoid issues // with magic property names like '__proto__'. The 'identifiers' object is used to share a single string instance for // each identifier in order to reduce memory consumption. function createIdentifier(isIdentifier: boolean, diagnosticMessage?: DiagnosticMessage): Identifier { identifierCount++; if (isIdentifier) { const node = createNode(SyntaxKind.Identifier); // Store original token kind if it is not just an Identifier so we can report appropriate error later in type checker if (token() !== SyntaxKind.Identifier) { node.originalKeywordKind = token(); } node.escapedText = escapeLeadingUnderscores(internIdentifier(scanner.getTokenValue())); nextToken(); return finishNode(node); } // Only for end of file because the error gets reported incorrectly on embedded script tags. const reportAtCurrentPosition = token() === SyntaxKind.EndOfFileToken; return createMissingNode(SyntaxKind.Identifier, reportAtCurrentPosition, diagnosticMessage || Diagnostics.Identifier_expected); } function parseIdentifier(diagnosticMessage?: DiagnosticMessage): Identifier { return createIdentifier(isIdentifier(), diagnosticMessage); } function parseIdentifierName(diagnosticMessage?: DiagnosticMessage): Identifier { return createIdentifier(tokenIsIdentifierOrKeyword(token()), diagnosticMessage); } function isLiteralPropertyName(): boolean { return tokenIsIdentifierOrKeyword(token()) || token() === SyntaxKind.StringLiteral || token() === SyntaxKind.NumericLiteral; } function parsePropertyNameWorker(allowComputedPropertyNames: boolean): PropertyName { if (token() === SyntaxKind.StringLiteral || token() === SyntaxKind.NumericLiteral) { const node = parseLiteralNode(); node.text = internIdentifier(node.text); return node; } if (allowComputedPropertyNames && token() === SyntaxKind.OpenBracketToken) { return parseComputedPropertyName(); } return parseIdentifierName(); } function parsePropertyName(): PropertyName { return parsePropertyNameWorker(/*allowComputedPropertyNames*/ true); } function parseComputedPropertyName(): ComputedPropertyName { // PropertyName [Yield]: // LiteralPropertyName // ComputedPropertyName[?Yield] const node = createNode(SyntaxKind.ComputedPropertyName); parseExpected(SyntaxKind.OpenBracketToken); // We parse any expression (including a comma expression). But the grammar // says that only an assignment expression is allowed, so the grammar checker // will error if it sees a comma expression. node.expression = allowInAnd(parseExpression); parseExpected(SyntaxKind.CloseBracketToken); return finishNode(node); } function parseContextualModifier(t: SyntaxKind): boolean { return token() === t && tryParse(nextTokenCanFollowModifier); } function nextTokenIsOnSameLineAndCanFollowModifier() { nextToken(); if (scanner.hasPrecedingLineBreak()) { return false; } return canFollowModifier(); } function nextTokenCanFollowModifier() { if (token() === SyntaxKind.ConstKeyword) { // 'const' is only a modifier if followed by 'enum'. return nextToken() === SyntaxKind.EnumKeyword; } if (token() === SyntaxKind.ExportKeyword) { nextToken(); if (token() === SyntaxKind.DefaultKeyword) { return lookAhead(nextTokenCanFollowDefaultKeyword); } return token() !== SyntaxKind.AsteriskToken && token() !== SyntaxKind.AsKeyword && token() !== SyntaxKind.OpenBraceToken && canFollowModifier(); } if (token() === SyntaxKind.DefaultKeyword) { return nextTokenCanFollowDefaultKeyword(); } if (token() === SyntaxKind.StaticKeyword) { nextToken(); return canFollowModifier(); } return nextTokenIsOnSameLineAndCanFollowModifier(); } function parseAnyContextualModifier(): boolean { return isModifierKind(token()) && tryParse(nextTokenCanFollowModifier); } function canFollowModifier(): boolean { return token() === SyntaxKind.OpenBracketToken || token() === SyntaxKind.OpenBraceToken || token() === SyntaxKind.AsteriskToken || token() === SyntaxKind.DotDotDotToken || isLiteralPropertyName(); } function nextTokenCanFollowDefaultKeyword(): boolean { nextToken(); return token() === SyntaxKind.ClassKeyword || token() === SyntaxKind.FunctionKeyword || token() === SyntaxKind.InterfaceKeyword || (token() === SyntaxKind.AbstractKeyword && lookAhead(nextTokenIsClassKeywordOnSameLine)) || (token() === SyntaxKind.AsyncKeyword && lookAhead(nextTokenIsFunctionKeywordOnSameLine)); } // True if positioned at the start of a list element function isListElement(parsingContext: ParsingContext, inErrorRecovery: boolean): boolean { const node = currentNode(parsingContext); if (node) { return true; } switch (parsingContext) { case ParsingContext.SourceElements: case ParsingContext.BlockStatements: case ParsingContext.SwitchClauseStatements: // If we're in error recovery, then we don't want to treat ';' as an empty statement. // The problem is that ';' can show up in far too many contexts, and if we see one // and assume it's a statement, then we may bail out inappropriately from whatever // we're parsing. For example, if we have a semicolon in the middle of a class, then // we really don't want to assume the class is over and we're on a statement in the // outer module. We just want to consume and move on. return !(token() === SyntaxKind.SemicolonToken && inErrorRecovery) && isStartOfStatement(); case ParsingContext.SwitchClauses: return token() === SyntaxKind.CaseKeyword || token() === SyntaxKind.DefaultKeyword; case ParsingContext.TypeMembers: return lookAhead(isTypeMemberStart); case ParsingContext.ClassMembers: // We allow semicolons as class elements (as specified by ES6) as long as we're // not in error recovery. If we're in error recovery, we don't want an errant // semicolon to be treated as a class member (since they're almost always used // for statements. return lookAhead(isClassMemberStart) || (token() === SyntaxKind.SemicolonToken && !inErrorRecovery); case ParsingContext.EnumMembers: // Include open bracket computed properties. This technically also lets in indexers, // which would be a candidate for improved error reporting. return token() === SyntaxKind.OpenBracketToken || isLiteralPropertyName(); case ParsingContext.ObjectLiteralMembers: return token() === SyntaxKind.OpenBracketToken || token() === SyntaxKind.AsteriskToken || token() === SyntaxKind.DotDotDotToken || isLiteralPropertyName(); case ParsingContext.RestProperties: return isLiteralPropertyName(); case ParsingContext.ObjectBindingElements: return token() === SyntaxKind.OpenBracketToken || token() === SyntaxKind.DotDotDotToken || isLiteralPropertyName(); case ParsingContext.HeritageClauseElement: // If we see `{ ... }` then only consume it as an expression if it is followed by `,` or `{` // That way we won't consume the body of a class in its heritage clause. if (token() === SyntaxKind.OpenBraceToken) { return lookAhead(isValidHeritageClauseObjectLiteral); } if (!inErrorRecovery) { return isStartOfLeftHandSideExpression() && !isHeritageClauseExtendsOrImplementsKeyword(); } else { // If we're in error recovery we tighten up what we're willing to match. // That way we don't treat something like "this" as a valid heritage clause // element during recovery. return isIdentifier() && !isHeritageClauseExtendsOrImplementsKeyword(); } case ParsingContext.VariableDeclarations: return isIdentifierOrPattern(); case ParsingContext.ArrayBindingElements: return token() === SyntaxKind.CommaToken || token() === SyntaxKind.DotDotDotToken || isIdentifierOrPattern(); case ParsingContext.TypeParameters: return isIdentifier(); case ParsingContext.ArrayLiteralMembers: if (token() === SyntaxKind.CommaToken) { return true; } // falls through case ParsingContext.ArgumentExpressions: return token() === SyntaxKind.DotDotDotToken || isStartOfExpression(); case ParsingContext.Parameters: return isStartOfParameter(); case ParsingContext.TypeArguments: case ParsingContext.TupleElementTypes: return token() === SyntaxKind.CommaToken || isStartOfType(); case ParsingContext.HeritageClauses: return isHeritageClause(); case ParsingContext.ImportOrExportSpecifiers: return tokenIsIdentifierOrKeyword(token()); case ParsingContext.JsxAttributes: return tokenIsIdentifierOrKeyword(token()) || token() === SyntaxKind.OpenBraceToken; case ParsingContext.JsxChildren: return true; } Debug.fail("Non-exhaustive case in 'isListElement'."); } function isValidHeritageClauseObjectLiteral() { Debug.assert(token() === SyntaxKind.OpenBraceToken); if (nextToken() === SyntaxKind.CloseBraceToken) { // if we see "extends {}" then only treat the {} as what we're extending (and not // the class body) if we have: // // extends {} { // extends {}, // extends {} extends // extends {} implements const next = nextToken(); return next === SyntaxKind.CommaToken || next === SyntaxKind.OpenBraceToken || next === SyntaxKind.ExtendsKeyword || next === SyntaxKind.ImplementsKeyword; } return true; } function nextTokenIsIdentifier() { nextToken(); return isIdentifier(); } function nextTokenIsIdentifierOrKeyword() { nextToken(); return tokenIsIdentifierOrKeyword(token()); } function nextTokenIsIdentifierOrKeywordOrGreaterThan() { nextToken(); return tokenIsIdentifierOrKeywordOrGreaterThan(token()); } function isHeritageClauseExtendsOrImplementsKeyword(): boolean { if (token() === SyntaxKind.ImplementsKeyword || token() === SyntaxKind.ExtendsKeyword) { return lookAhead(nextTokenIsStartOfExpression); } return false; } function nextTokenIsStartOfExpression() { nextToken(); return isStartOfExpression(); } function nextTokenIsStartOfType() { nextToken(); return isStartOfType(); } // True if positioned at a list terminator function isListTerminator(kind: ParsingContext): boolean { if (token() === SyntaxKind.EndOfFileToken) { // Being at the end of the file ends all lists. return true; } switch (kind) { case ParsingContext.BlockStatements: case ParsingContext.SwitchClauses: case ParsingContext.TypeMembers: case ParsingContext.ClassMembers: case ParsingContext.EnumMembers: case ParsingContext.ObjectLiteralMembers: case ParsingContext.ObjectBindingElements: case ParsingContext.ImportOrExportSpecifiers: return token() === SyntaxKind.CloseBraceToken; case ParsingContext.SwitchClauseStatements: return token() === SyntaxKind.CloseBraceToken || token() === SyntaxKind.CaseKeyword || token() === SyntaxKind.DefaultKeyword; case ParsingContext.HeritageClauseElement: return token() === SyntaxKind.OpenBraceToken || token() === SyntaxKind.ExtendsKeyword || token() === SyntaxKind.ImplementsKeyword; case ParsingContext.VariableDeclarations: return isVariableDeclaratorListTerminator(); case ParsingContext.TypeParameters: // Tokens other than '>' are here for better error recovery return token() === SyntaxKind.GreaterThanToken || token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.OpenBraceToken || token() === SyntaxKind.ExtendsKeyword || token() === SyntaxKind.ImplementsKeyword; case ParsingContext.ArgumentExpressions: // Tokens other than ')' are here for better error recovery return token() === SyntaxKind.CloseParenToken || token() === SyntaxKind.SemicolonToken; case ParsingContext.ArrayLiteralMembers: case ParsingContext.TupleElementTypes: case ParsingContext.ArrayBindingElements: return token() === SyntaxKind.CloseBracketToken; case ParsingContext.Parameters: case ParsingContext.RestProperties: // Tokens other than ')' and ']' (the latter for index signatures) are here for better error recovery return token() === SyntaxKind.CloseParenToken || token() === SyntaxKind.CloseBracketToken /*|| token === SyntaxKind.OpenBraceToken*/; case ParsingContext.TypeArguments: // All other tokens should cause the type-argument to terminate except comma token return token() !== SyntaxKind.CommaToken; case ParsingContext.HeritageClauses: return token() === SyntaxKind.OpenBraceToken || token() === SyntaxKind.CloseBraceToken; case ParsingContext.JsxAttributes: return token() === SyntaxKind.GreaterThanToken || token() === SyntaxKind.SlashToken; case ParsingContext.JsxChildren: return token() === SyntaxKind.LessThanToken && lookAhead(nextTokenIsSlash); } } function isVariableDeclaratorListTerminator(): boolean { // If we can consume a semicolon (either explicitly, or with ASI), then consider us done // with parsing the list of variable declarators. if (canParseSemicolon()) { return true; } // in the case where we're parsing the variable declarator of a 'for-in' statement, we // are done if we see an 'in' keyword in front of us. Same with for-of if (isInOrOfKeyword(token())) { return true; } // ERROR RECOVERY TWEAK: // For better error recovery, if we see an '=>' then we just stop immediately. We've got an // arrow function here and it's going to be very unlikely that we'll resynchronize and get // another variable declaration. if (token() === SyntaxKind.EqualsGreaterThanToken) { return true; } // Keep trying to parse out variable declarators. return false; } // True if positioned at element or terminator of the current list or any enclosing list function isInSomeParsingContext(): boolean { for (let kind = 0; kind < ParsingContext.Count; kind++) { if (parsingContext & (1 << kind)) { if (isListElement(kind, /*inErrorRecovery*/ true) || isListTerminator(kind)) { return true; } } } return false; } // Parses a list of elements function parseList(kind: ParsingContext, parseElement: () => T): NodeArray { const saveParsingContext = parsingContext; parsingContext |= 1 << kind; const list = []; const listPos = getNodePos(); while (!isListTerminator(kind)) { if (isListElement(kind, /*inErrorRecovery*/ false)) { const element = parseListElement(kind, parseElement); list.push(element); continue; } if (abortParsingListOrMoveToNextToken(kind)) { break; } } parsingContext = saveParsingContext; return createNodeArray(list, listPos); } function parseListElement(parsingContext: ParsingContext, parseElement: () => T): T { const node = currentNode(parsingContext); if (node) { return consumeNode(node); } return parseElement(); } function currentNode(parsingContext: ParsingContext): Node { // If there is an outstanding parse error that we've encountered, but not attached to // some node, then we cannot get a node from the old source tree. This is because we // want to mark the next node we encounter as being unusable. // // Note: This may be too conservative. Perhaps we could reuse the node and set the bit // on it (or its leftmost child) as having the error. For now though, being conservative // is nice and likely won't ever affect perf. if (parseErrorBeforeNextFinishedNode) { return undefined; } if (!syntaxCursor) { // if we don't have a cursor, we could never return a node from the old tree. return undefined; } const node = syntaxCursor.currentNode(scanner.getStartPos()); // Can't reuse a missing node. if (nodeIsMissing(node)) { return undefined; } // Can't reuse a node that intersected the change range. if (node.intersectsChange) { return undefined; } // Can't reuse a node that contains a parse error. This is necessary so that we // produce the same set of errors again. if (containsParseError(node)) { return undefined; } // We can only reuse a node if it was parsed under the same strict mode that we're // currently in. i.e. if we originally parsed a node in non-strict mode, but then // the user added 'using strict' at the top of the file, then we can't use that node // again as the presence of strict mode may cause us to parse the tokens in the file // differently. // // Note: we *can* reuse tokens when the strict mode changes. That's because tokens // are unaffected by strict mode. It's just the parser will decide what to do with it // differently depending on what mode it is in. // // This also applies to all our other context flags as well. const nodeContextFlags = node.flags & NodeFlags.ContextFlags; if (nodeContextFlags !== contextFlags) { return undefined; } // Ok, we have a node that looks like it could be reused. Now verify that it is valid // in the current list parsing context that we're currently at. if (!canReuseNode(node, parsingContext)) { return undefined; } if ((node as JSDocContainer).jsDocCache) { // jsDocCache may include tags from parent nodes, which might have been modified. (node as JSDocContainer).jsDocCache = undefined; } return node; } function consumeNode(node: Node) { // Move the scanner so it is after the node we just consumed. scanner.setTextPos(node.end); nextToken(); return node; } function canReuseNode(node: Node, parsingContext: ParsingContext): boolean { switch (parsingContext) { case ParsingContext.ClassMembers: return isReusableClassMember(node); case ParsingContext.SwitchClauses: return isReusableSwitchClause(node); case ParsingContext.SourceElements: case ParsingContext.BlockStatements: case ParsingContext.SwitchClauseStatements: return isReusableStatement(node); case ParsingContext.EnumMembers: return isReusableEnumMember(node); case ParsingContext.TypeMembers: return isReusableTypeMember(node); case ParsingContext.VariableDeclarations: return isReusableVariableDeclaration(node); case ParsingContext.Parameters: return isReusableParameter(node); case ParsingContext.RestProperties: return false; // Any other lists we do not care about reusing nodes in. But feel free to add if // you can do so safely. Danger areas involve nodes that may involve speculative // parsing. If speculative parsing is involved with the node, then the range the // parser reached while looking ahead might be in the edited range (see the example // in canReuseVariableDeclaratorNode for a good case of this). case ParsingContext.HeritageClauses: // This would probably be safe to reuse. There is no speculative parsing with // heritage clauses. case ParsingContext.TypeParameters: // This would probably be safe to reuse. There is no speculative parsing with // type parameters. Note that that's because type *parameters* only occur in // unambiguous *type* contexts. While type *arguments* occur in very ambiguous // *expression* contexts. case ParsingContext.TupleElementTypes: // This would probably be safe to reuse. There is no speculative parsing with // tuple types. // Technically, type argument list types are probably safe to reuse. While // speculative parsing is involved with them (since type argument lists are only // produced from speculative parsing a < as a type argument list), we only have // the types because speculative parsing succeeded. Thus, the lookahead never // went past the end of the list and rewound. case ParsingContext.TypeArguments: // Note: these are almost certainly not safe to ever reuse. Expressions commonly // need a large amount of lookahead, and we should not reuse them as they may // have actually intersected the edit. case ParsingContext.ArgumentExpressions: // This is not safe to reuse for the same reason as the 'AssignmentExpression' // cases. i.e. a property assignment may end with an expression, and thus might // have lookahead far beyond it's old node. case ParsingContext.ObjectLiteralMembers: // This is probably not safe to reuse. There can be speculative parsing with // type names in a heritage clause. There can be generic names in the type // name list, and there can be left hand side expressions (which can have type // arguments.) case ParsingContext.HeritageClauseElement: // Perhaps safe to reuse, but it's unlikely we'd see more than a dozen attributes // on any given element. Same for children. case ParsingContext.JsxAttributes: case ParsingContext.JsxChildren: } return false; } function isReusableClassMember(node: Node) { if (node) { switch (node.kind) { case SyntaxKind.Constructor: case SyntaxKind.IndexSignature: case SyntaxKind.GetAccessor: case SyntaxKind.SetAccessor: case SyntaxKind.PropertyDeclaration: case SyntaxKind.SemicolonClassElement: return true; case SyntaxKind.MethodDeclaration: // Method declarations are not necessarily reusable. An object-literal // may have a method calls "constructor(...)" and we must reparse that // into an actual .ConstructorDeclaration. const methodDeclaration = node; const nameIsConstructor = methodDeclaration.name.kind === SyntaxKind.Identifier && (methodDeclaration.name).originalKeywordKind === SyntaxKind.ConstructorKeyword; return !nameIsConstructor; } } return false; } function isReusableSwitchClause(node: Node) { if (node) { switch (node.kind) { case SyntaxKind.CaseClause: case SyntaxKind.DefaultClause: return true; } } return false; } function isReusableStatement(node: Node) { if (node) { switch (node.kind) { case SyntaxKind.FunctionDeclaration: case SyntaxKind.VariableStatement: case SyntaxKind.Block: case SyntaxKind.IfStatement: case SyntaxKind.ExpressionStatement: case SyntaxKind.ThrowStatement: case SyntaxKind.ReturnStatement: case SyntaxKind.SwitchStatement: case SyntaxKind.BreakStatement: case SyntaxKind.ContinueStatement: case SyntaxKind.ForInStatement: case SyntaxKind.ForOfStatement: case SyntaxKind.ForStatement: case SyntaxKind.WhileStatement: case SyntaxKind.WithStatement: case SyntaxKind.EmptyStatement: case SyntaxKind.TryStatement: case SyntaxKind.LabeledStatement: case SyntaxKind.DoStatement: case SyntaxKind.DebuggerStatement: case SyntaxKind.ImportDeclaration: case SyntaxKind.ImportEqualsDeclaration: case SyntaxKind.ExportDeclaration: case SyntaxKind.ExportAssignment: case SyntaxKind.ModuleDeclaration: case SyntaxKind.ClassDeclaration: case SyntaxKind.InterfaceDeclaration: case SyntaxKind.EnumDeclaration: case SyntaxKind.TypeAliasDeclaration: return true; } } return false; } function isReusableEnumMember(node: Node) { return node.kind === SyntaxKind.EnumMember; } function isReusableTypeMember(node: Node) { if (node) { switch (node.kind) { case SyntaxKind.ConstructSignature: case SyntaxKind.MethodSignature: case SyntaxKind.IndexSignature: case SyntaxKind.PropertySignature: case SyntaxKind.CallSignature: return true; } } return false; } function isReusableVariableDeclaration(node: Node) { if (node.kind !== SyntaxKind.VariableDeclaration) { return false; } // Very subtle incremental parsing bug. Consider the following code: // // let v = new List < A, B // // This is actually legal code. It's a list of variable declarators "v = new List() // // then we have a problem. "v = new Listnode; return variableDeclarator.initializer === undefined; } function isReusableParameter(node: Node) { if (node.kind !== SyntaxKind.Parameter) { return false; } // See the comment in isReusableVariableDeclaration for why we do this. const parameter = node; return parameter.initializer === undefined; } // Returns true if we should abort parsing. function abortParsingListOrMoveToNextToken(kind: ParsingContext) { parseErrorAtCurrentToken(parsingContextErrors(kind)); if (isInSomeParsingContext()) { return true; } nextToken(); return false; } function parsingContextErrors(context: ParsingContext): DiagnosticMessage { switch (context) { case ParsingContext.SourceElements: return Diagnostics.Declaration_or_statement_expected; case ParsingContext.BlockStatements: return Diagnostics.Declaration_or_statement_expected; case ParsingContext.SwitchClauses: return Diagnostics.case_or_default_expected; case ParsingContext.SwitchClauseStatements: return Diagnostics.Statement_expected; case ParsingContext.RestProperties: // fallthrough case ParsingContext.TypeMembers: return Diagnostics.Property_or_signature_expected; case ParsingContext.ClassMembers: return Diagnostics.Unexpected_token_A_constructor_method_accessor_or_property_was_expected; case ParsingContext.EnumMembers: return Diagnostics.Enum_member_expected; case ParsingContext.HeritageClauseElement: return Diagnostics.Expression_expected; case ParsingContext.VariableDeclarations: return Diagnostics.Variable_declaration_expected; case ParsingContext.ObjectBindingElements: return Diagnostics.Property_destructuring_pattern_expected; case ParsingContext.ArrayBindingElements: return Diagnostics.Array_element_destructuring_pattern_expected; case ParsingContext.ArgumentExpressions: return Diagnostics.Argument_expression_expected; case ParsingContext.ObjectLiteralMembers: return Diagnostics.Property_assignment_expected; case ParsingContext.ArrayLiteralMembers: return Diagnostics.Expression_or_comma_expected; case ParsingContext.Parameters: return Diagnostics.Parameter_declaration_expected; case ParsingContext.TypeParameters: return Diagnostics.Type_parameter_declaration_expected; case ParsingContext.TypeArguments: return Diagnostics.Type_argument_expected; case ParsingContext.TupleElementTypes: return Diagnostics.Type_expected; case ParsingContext.HeritageClauses: return Diagnostics.Unexpected_token_expected; case ParsingContext.ImportOrExportSpecifiers: return Diagnostics.Identifier_expected; case ParsingContext.JsxAttributes: return Diagnostics.Identifier_expected; case ParsingContext.JsxChildren: return Diagnostics.Identifier_expected; } } // Parses a comma-delimited list of elements function parseDelimitedList(kind: ParsingContext, parseElement: () => T, considerSemicolonAsDelimiter?: boolean): NodeArray { const saveParsingContext = parsingContext; parsingContext |= 1 << kind; const list = []; const listPos = getNodePos(); let commaStart = -1; // Meaning the previous token was not a comma while (true) { if (isListElement(kind, /*inErrorRecovery*/ false)) { const startPos = scanner.getStartPos(); list.push(parseListElement(kind, parseElement)); commaStart = scanner.getTokenPos(); if (parseOptional(SyntaxKind.CommaToken)) { // No need to check for a zero length node since we know we parsed a comma continue; } commaStart = -1; // Back to the state where the last token was not a comma if (isListTerminator(kind)) { break; } // We didn't get a comma, and the list wasn't terminated, explicitly parse // out a comma so we give a good error message. parseExpected(SyntaxKind.CommaToken); // If the token was a semicolon, and the caller allows that, then skip it and // continue. This ensures we get back on track and don't result in tons of // parse errors. For example, this can happen when people do things like use // a semicolon to delimit object literal members. Note: we'll have already // reported an error when we called parseExpected above. if (considerSemicolonAsDelimiter && token() === SyntaxKind.SemicolonToken && !scanner.hasPrecedingLineBreak()) { nextToken(); } if (startPos === scanner.getStartPos()) { // What we're parsing isn't actually remotely recognizable as a element and we've consumed no tokens whatsoever // Consume a token to advance the parser in some way and avoid an infinite loop // This can happen when we're speculatively parsing parenthesized expressions which we think may be arrow functions, // or when a modifier keyword which is disallowed as a parameter name (ie, `static` in strict mode) is supplied nextToken(); } continue; } if (isListTerminator(kind)) { break; } if (abortParsingListOrMoveToNextToken(kind)) { break; } } parsingContext = saveParsingContext; const result = createNodeArray(list, listPos); // Recording the trailing comma is deliberately done after the previous // loop, and not just if we see a list terminator. This is because the list // may have ended incorrectly, but it is still important to know if there // was a trailing comma. // Check if the last token was a comma. if (commaStart >= 0) { // Always preserve a trailing comma by marking it on the NodeArray result.hasTrailingComma = true; } return result; } function createMissingList(): NodeArray { return createNodeArray([], getNodePos()); } function parseBracketedList(kind: ParsingContext, parseElement: () => T, open: SyntaxKind, close: SyntaxKind): NodeArray { if (parseExpected(open)) { const result = parseDelimitedList(kind, parseElement); parseExpected(close); return result; } return createMissingList(); } function parseEntityName(allowReservedWords: boolean, diagnosticMessage?: DiagnosticMessage): EntityName { let entity: EntityName = allowReservedWords ? parseIdentifierName(diagnosticMessage) : parseIdentifier(diagnosticMessage); let dotPos = scanner.getStartPos(); while (parseOptional(SyntaxKind.DotToken)) { if (token() === SyntaxKind.LessThanToken) { // the entity is part of a JSDoc-style generic, so record the trailing dot for later error reporting entity.jsdocDotPos = dotPos; break; } dotPos = scanner.getStartPos(); entity = createQualifiedName(entity, parseRightSideOfDot(allowReservedWords)); } return entity; } function createQualifiedName(entity: EntityName, name: Identifier): QualifiedName { const node = createNode(SyntaxKind.QualifiedName, entity.pos) as QualifiedName; node.left = entity; node.right = name; return finishNode(node); } function parseRightSideOfDot(allowIdentifierNames: boolean): Identifier { // Technically a keyword is valid here as all identifiers and keywords are identifier names. // However, often we'll encounter this in error situations when the identifier or keyword // is actually starting another valid construct. // // So, we check for the following specific case: // // name. // identifierOrKeyword identifierNameOrKeyword // // Note: the newlines are important here. For example, if that above code // were rewritten into: // // name.identifierOrKeyword // identifierNameOrKeyword // // Then we would consider it valid. That's because ASI would take effect and // the code would be implicitly: "name.identifierOrKeyword; identifierNameOrKeyword". // In the first case though, ASI will not take effect because there is not a // line terminator after the identifier or keyword. if (scanner.hasPrecedingLineBreak() && tokenIsIdentifierOrKeyword(token())) { const matchesPattern = lookAhead(nextTokenIsIdentifierOrKeywordOnSameLine); if (matchesPattern) { // Report that we need an identifier. However, report it right after the dot, // and not on the next token. This is because the next token might actually // be an identifier and the error would be quite confusing. return createMissingNode(SyntaxKind.Identifier, /*reportAtCurrentPosition*/ true, Diagnostics.Identifier_expected); } } return allowIdentifierNames ? parseIdentifierName() : parseIdentifier(); } function parseTemplateExpression(): TemplateExpression { const template = createNode(SyntaxKind.TemplateExpression); template.head = parseTemplateHead(); Debug.assert(template.head.kind === SyntaxKind.TemplateHead, "Template head has wrong token kind"); const list = []; const listPos = getNodePos(); do { list.push(parseTemplateSpan()); } while (lastOrUndefined(list).literal.kind === SyntaxKind.TemplateMiddle); template.templateSpans = createNodeArray(list, listPos); return finishNode(template); } function parseTemplateSpan(): TemplateSpan { const span = createNode(SyntaxKind.TemplateSpan); span.expression = allowInAnd(parseExpression); let literal: TemplateMiddle | TemplateTail; if (token() === SyntaxKind.CloseBraceToken) { reScanTemplateToken(); literal = parseTemplateMiddleOrTemplateTail(); } else { literal = parseExpectedToken(SyntaxKind.TemplateTail, Diagnostics._0_expected, tokenToString(SyntaxKind.CloseBraceToken)); } span.literal = literal; return finishNode(span); } function parseLiteralNode(): LiteralExpression { return parseLiteralLikeNode(token()); } function parseTemplateHead(): TemplateHead { const fragment = parseLiteralLikeNode(token()); Debug.assert(fragment.kind === SyntaxKind.TemplateHead, "Template head has wrong token kind"); return fragment; } function parseTemplateMiddleOrTemplateTail(): TemplateMiddle | TemplateTail { const fragment = parseLiteralLikeNode(token()); Debug.assert(fragment.kind === SyntaxKind.TemplateMiddle || fragment.kind === SyntaxKind.TemplateTail, "Template fragment has wrong token kind"); return fragment; } function parseLiteralLikeNode(kind: SyntaxKind): LiteralExpression | LiteralLikeNode { const node = createNode(kind); const text = scanner.getTokenValue(); node.text = text; if (scanner.hasExtendedUnicodeEscape()) { node.hasExtendedUnicodeEscape = true; } if (scanner.isUnterminated()) { node.isUnterminated = true; } // Octal literals are not allowed in strict mode or ES5 // Note that theoretically the following condition would hold true literals like 009, // which is not octal.But because of how the scanner separates the tokens, we would // never get a token like this. Instead, we would get 00 and 9 as two separate tokens. // We also do not need to check for negatives because any prefix operator would be part of a // parent unary expression. if (node.kind === SyntaxKind.NumericLiteral) { (node).numericLiteralFlags = scanner.getTokenFlags() & TokenFlags.NumericLiteralFlags; } nextToken(); finishNode(node); return node; } // TYPES function parseTypeReference(): TypeReferenceNode { const node = createNode(SyntaxKind.TypeReference); node.typeName = parseEntityName(/*allowReservedWords*/ true, Diagnostics.Type_expected); if (!scanner.hasPrecedingLineBreak() && token() === SyntaxKind.LessThanToken) { node.typeArguments = parseBracketedList(ParsingContext.TypeArguments, parseType, SyntaxKind.LessThanToken, SyntaxKind.GreaterThanToken); } return finishNode(node); } function parseThisTypePredicate(lhs: ThisTypeNode): TypePredicateNode { nextToken(); const node = createNode(SyntaxKind.TypePredicate, lhs.pos) as TypePredicateNode; node.parameterName = lhs; node.type = parseType(); return finishNode(node); } function parseThisTypeNode(): ThisTypeNode { const node = createNode(SyntaxKind.ThisType) as ThisTypeNode; nextToken(); return finishNode(node); } function parseJSDocAllType(): JSDocAllType { const result = createNode(SyntaxKind.JSDocAllType); nextToken(); return finishNode(result); } function parseJSDocUnknownOrNullableType(): JSDocUnknownType | JSDocNullableType { const pos = scanner.getStartPos(); // skip the ? nextToken(); // Need to lookahead to decide if this is a nullable or unknown type. // Here are cases where we'll pick the unknown type: // // Foo(?, // { a: ? } // Foo(?) // Foo // Foo(?= // (?| if (token() === SyntaxKind.CommaToken || token() === SyntaxKind.CloseBraceToken || token() === SyntaxKind.CloseParenToken || token() === SyntaxKind.GreaterThanToken || token() === SyntaxKind.EqualsToken || token() === SyntaxKind.BarToken) { const result = createNode(SyntaxKind.JSDocUnknownType, pos); return finishNode(result); } else { const result = createNode(SyntaxKind.JSDocNullableType, pos); result.type = parseType(); return finishNode(result); } } function parseJSDocFunctionType(): JSDocFunctionType | TypeReferenceNode { if (lookAhead(nextTokenIsOpenParen)) { const result = createNodeWithJSDoc(SyntaxKind.JSDocFunctionType); nextToken(); fillSignature(SyntaxKind.ColonToken, SignatureFlags.Type | SignatureFlags.JSDoc, result); return finishNode(result); } const node = createNode(SyntaxKind.TypeReference); node.typeName = parseIdentifierName(); return finishNode(node); } function parseJSDocParameter(): ParameterDeclaration { const parameter = createNode(SyntaxKind.Parameter) as ParameterDeclaration; if (token() === SyntaxKind.ThisKeyword || token() === SyntaxKind.NewKeyword) { parameter.name = parseIdentifierName(); parseExpected(SyntaxKind.ColonToken); } parameter.type = parseType(); return finishNode(parameter); } function parseJSDocNodeWithType(kind: SyntaxKind.JSDocVariadicType | SyntaxKind.JSDocNonNullableType): TypeNode { const result = createNode(kind) as JSDocVariadicType | JSDocNonNullableType; nextToken(); result.type = parseNonArrayType(); return finishNode(result); } function parseTypeQuery(): TypeQueryNode { const node = createNode(SyntaxKind.TypeQuery); parseExpected(SyntaxKind.TypeOfKeyword); node.exprName = parseEntityName(/*allowReservedWords*/ true); return finishNode(node); } function parseTypeParameter(): TypeParameterDeclaration { const node = createNode(SyntaxKind.TypeParameter); node.name = parseIdentifier(); if (parseOptional(SyntaxKind.ExtendsKeyword)) { // It's not uncommon for people to write improper constraints to a generic. If the // user writes a constraint that is an expression and not an actual type, then parse // it out as an expression (so we can recover well), but report that a type is needed // instead. if (isStartOfType() || !isStartOfExpression()) { node.constraint = parseType(); } else { // It was not a type, and it looked like an expression. Parse out an expression // here so we recover well. Note: it is important that we call parseUnaryExpression // and not parseExpression here. If the user has: // // // // We do *not* want to consume the `>` as we're consuming the expression for "". node.expression = parseUnaryExpressionOrHigher(); } } if (parseOptional(SyntaxKind.EqualsToken)) { node.default = parseType(); } return finishNode(node); } function parseTypeParameters(): NodeArray | undefined { if (token() === SyntaxKind.LessThanToken) { return parseBracketedList(ParsingContext.TypeParameters, parseTypeParameter, SyntaxKind.LessThanToken, SyntaxKind.GreaterThanToken); } } function parseParameterType(): TypeNode { if (parseOptional(SyntaxKind.ColonToken)) { return parseType(); } return undefined; } function isStartOfParameter(): boolean { return token() === SyntaxKind.DotDotDotToken || isIdentifierOrPattern() || isModifierKind(token()) || token() === SyntaxKind.AtToken || isStartOfType(/*inStartOfParameter*/ true); } function parseParameter(): ParameterDeclaration { const node = createNodeWithJSDoc(SyntaxKind.Parameter); if (token() === SyntaxKind.ThisKeyword) { node.name = createIdentifier(/*isIdentifier*/ true); node.type = parseParameterType(); return finishNode(node); } node.decorators = parseDecorators(); node.modifiers = parseModifiers(); node.dotDotDotToken = parseOptionalToken(SyntaxKind.DotDotDotToken); // FormalParameter [Yield,Await]: // BindingElement[?Yield,?Await] node.name = parseIdentifierOrPattern(); if (getFullWidth(node.name) === 0 && !hasModifiers(node) && isModifierKind(token())) { // in cases like // 'use strict' // function foo(static) // isParameter('static') === true, because of isModifier('static') // however 'static' is not a legal identifier in a strict mode. // so result of this function will be ParameterDeclaration (flags = 0, name = missing, type = undefined, initializer = undefined) // and current token will not change => parsing of the enclosing parameter list will last till the end of time (or OOM) // to avoid this we'll advance cursor to the next token. nextToken(); } node.questionToken = parseOptionalToken(SyntaxKind.QuestionToken); node.type = parseParameterType(); node.initializer = parseInitializer(); return finishNode(node); } function fillSignature( returnToken: SyntaxKind.ColonToken | SyntaxKind.EqualsGreaterThanToken, flags: SignatureFlags, signature: SignatureDeclaration): void { if (!(flags & SignatureFlags.JSDoc)) { signature.typeParameters = parseTypeParameters(); } signature.parameters = parseParameterList(flags); signature.type = parseReturnType(returnToken, !!(flags & SignatureFlags.Type)); } function parseReturnType(returnToken: SyntaxKind.ColonToken | SyntaxKind.EqualsGreaterThanToken, isType: boolean): TypeNode | undefined { return shouldParseReturnType(returnToken, isType) ? parseTypeOrTypePredicate() : undefined; } function shouldParseReturnType(returnToken: SyntaxKind.ColonToken | SyntaxKind.EqualsGreaterThanToken, isType: boolean): boolean { if (returnToken === SyntaxKind.EqualsGreaterThanToken) { parseExpected(returnToken); return true; } else if (parseOptional(SyntaxKind.ColonToken)) { return true; } else if (isType && token() === SyntaxKind.EqualsGreaterThanToken) { // This is easy to get backward, especially in type contexts, so parse the type anyway parseErrorAtCurrentToken(Diagnostics._0_expected, tokenToString(SyntaxKind.ColonToken)); nextToken(); return true; } return false; } function parseParameterList(flags: SignatureFlags) { // FormalParameters [Yield,Await]: (modified) // [empty] // FormalParameterList[?Yield,Await] // // FormalParameter[Yield,Await]: (modified) // BindingElement[?Yield,Await] // // BindingElement [Yield,Await]: (modified) // SingleNameBinding[?Yield,?Await] // BindingPattern[?Yield,?Await]Initializer [In, ?Yield,?Await] opt // // SingleNameBinding [Yield,Await]: // BindingIdentifier[?Yield,?Await]Initializer [In, ?Yield,?Await] opt if (parseExpected(SyntaxKind.OpenParenToken)) { const savedYieldContext = inYieldContext(); const savedAwaitContext = inAwaitContext(); setYieldContext(!!(flags & SignatureFlags.Yield)); setAwaitContext(!!(flags & SignatureFlags.Await)); const result = parseDelimitedList(ParsingContext.Parameters, flags & SignatureFlags.JSDoc ? parseJSDocParameter : parseParameter); setYieldContext(savedYieldContext); setAwaitContext(savedAwaitContext); if (!parseExpected(SyntaxKind.CloseParenToken) && (flags & SignatureFlags.RequireCompleteParameterList)) { // Caller insisted that we had to end with a ) We didn't. So just return // undefined here. return undefined; } return result; } // We didn't even have an open paren. If the caller requires a complete parameter list, // we definitely can't provide that. However, if they're ok with an incomplete one, // then just return an empty set of parameters. return (flags & SignatureFlags.RequireCompleteParameterList) ? undefined : createMissingList(); } function parseTypeMemberSemicolon() { // We allow type members to be separated by commas or (possibly ASI) semicolons. // First check if it was a comma. If so, we're done with the member. if (parseOptional(SyntaxKind.CommaToken)) { return; } // Didn't have a comma. We must have a (possible ASI) semicolon. parseSemicolon(); } function parseSignatureMember(kind: SyntaxKind.CallSignature | SyntaxKind.ConstructSignature): CallSignatureDeclaration | ConstructSignatureDeclaration { const node = createNodeWithJSDoc(kind); if (kind === SyntaxKind.ConstructSignature) { parseExpected(SyntaxKind.NewKeyword); } fillSignature(SyntaxKind.ColonToken, SignatureFlags.Type, node); parseTypeMemberSemicolon(); return finishNode(node); } function isIndexSignature(): boolean { return token() === SyntaxKind.OpenBracketToken && lookAhead(isUnambiguouslyIndexSignature); } function isUnambiguouslyIndexSignature() { // The only allowed sequence is: // // [id: // // However, for error recovery, we also check the following cases: // // [... // [id, // [id?, // [id?: // [id?] // [public id // [private id // [protected id // [] // nextToken(); if (token() === SyntaxKind.DotDotDotToken || token() === SyntaxKind.CloseBracketToken) { return true; } if (isModifierKind(token())) { nextToken(); if (isIdentifier()) { return true; } } else if (!isIdentifier()) { return false; } else { // Skip the identifier nextToken(); } // A colon signifies a well formed indexer // A comma should be a badly formed indexer because comma expressions are not allowed // in computed properties. if (token() === SyntaxKind.ColonToken || token() === SyntaxKind.CommaToken) { return true; } // Question mark could be an indexer with an optional property, // or it could be a conditional expression in a computed property. if (token() !== SyntaxKind.QuestionToken) { return false; } // If any of the following tokens are after the question mark, it cannot // be a conditional expression, so treat it as an indexer. nextToken(); return token() === SyntaxKind.ColonToken || token() === SyntaxKind.CommaToken || token() === SyntaxKind.CloseBracketToken; } function parseIndexSignatureDeclaration(node: IndexSignatureDeclaration): IndexSignatureDeclaration { node.kind = SyntaxKind.IndexSignature; node.parameters = parseBracketedList(ParsingContext.Parameters, parseParameter, SyntaxKind.OpenBracketToken, SyntaxKind.CloseBracketToken); node.type = parseTypeAnnotation(); parseTypeMemberSemicolon(); return finishNode(node); } function parsePropertyOrMethodSignature(node: PropertySignature | MethodSignature): PropertySignature | MethodSignature { node.name = parsePropertyName(); node.questionToken = parseOptionalToken(SyntaxKind.QuestionToken); if (token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.LessThanToken) { node.kind = SyntaxKind.MethodSignature; // Method signatures don't exist in expression contexts. So they have neither // [Yield] nor [Await] fillSignature(SyntaxKind.ColonToken, SignatureFlags.Type, node); } else { node.kind = SyntaxKind.PropertySignature; node.type = parseTypeAnnotation(); if (token() === SyntaxKind.EqualsToken) { // Although type literal properties cannot not have initializers, we attempt // to parse an initializer so we can report in the checker that an interface // property or type literal property cannot have an initializer. (node).initializer = parseInitializer(); } } parseTypeMemberSemicolon(); return finishNode(node); } function isTypeMemberStart(): boolean { // Return true if we have the start of a signature member if (token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.LessThanToken) { return true; } let idToken: boolean; // Eat up all modifiers, but hold on to the last one in case it is actually an identifier while (isModifierKind(token())) { idToken = true; nextToken(); } // Index signatures and computed property names are type members if (token() === SyntaxKind.OpenBracketToken) { return true; } // Try to get the first property-like token following all modifiers if (isLiteralPropertyName()) { idToken = true; nextToken(); } // If we were able to get any potential identifier, check that it is // the start of a member declaration if (idToken) { return token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.LessThanToken || token() === SyntaxKind.QuestionToken || token() === SyntaxKind.ColonToken || token() === SyntaxKind.CommaToken || canParseSemicolon(); } return false; } function parseTypeMember(): TypeElement { if (token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.LessThanToken) { return parseSignatureMember(SyntaxKind.CallSignature); } if (token() === SyntaxKind.NewKeyword && lookAhead(nextTokenIsOpenParenOrLessThan)) { return parseSignatureMember(SyntaxKind.ConstructSignature); } const node = createNodeWithJSDoc(SyntaxKind.Unknown); node.modifiers = parseModifiers(); if (isIndexSignature()) { return parseIndexSignatureDeclaration(node); } return parsePropertyOrMethodSignature(node); } function nextTokenIsOpenParenOrLessThan() { nextToken(); return token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.LessThanToken; } function parseTypeLiteral(): TypeLiteralNode { const node = createNode(SyntaxKind.TypeLiteral); node.members = parseObjectTypeMembers(); return finishNode(node); } function parseObjectTypeMembers(): NodeArray { let members: NodeArray; if (parseExpected(SyntaxKind.OpenBraceToken)) { members = parseList(ParsingContext.TypeMembers, parseTypeMember); parseExpected(SyntaxKind.CloseBraceToken); } else { members = createMissingList(); } return members; } function isStartOfMappedType() { nextToken(); if (token() === SyntaxKind.PlusToken || token() === SyntaxKind.MinusToken) { return nextToken() === SyntaxKind.ReadonlyKeyword; } if (token() === SyntaxKind.ReadonlyKeyword) { nextToken(); } return token() === SyntaxKind.OpenBracketToken && nextTokenIsIdentifier() && nextToken() === SyntaxKind.InKeyword; } function parseMappedTypeParameter() { const node = createNode(SyntaxKind.TypeParameter); node.name = parseIdentifier(); parseExpected(SyntaxKind.InKeyword); node.constraint = parseType(); return finishNode(node); } function parseMappedType() { const node = createNode(SyntaxKind.MappedType); parseExpected(SyntaxKind.OpenBraceToken); if (token() === SyntaxKind.ReadonlyKeyword || token() === SyntaxKind.PlusToken || token() === SyntaxKind.MinusToken) { node.readonlyToken = parseTokenNode(); if (node.readonlyToken.kind !== SyntaxKind.ReadonlyKeyword) { parseExpectedToken(SyntaxKind.ReadonlyKeyword); } } parseExpected(SyntaxKind.OpenBracketToken); node.typeParameter = parseMappedTypeParameter(); parseExpected(SyntaxKind.CloseBracketToken); if (token() === SyntaxKind.QuestionToken || token() === SyntaxKind.PlusToken || token() === SyntaxKind.MinusToken) { node.questionToken = parseTokenNode(); if (node.questionToken.kind !== SyntaxKind.QuestionToken) { parseExpectedToken(SyntaxKind.QuestionToken); } } node.type = parseTypeAnnotation(); parseSemicolon(); parseExpected(SyntaxKind.CloseBraceToken); return finishNode(node); } function parseTupleType(): TupleTypeNode { const node = createNode(SyntaxKind.TupleType); node.elementTypes = parseBracketedList(ParsingContext.TupleElementTypes, parseType, SyntaxKind.OpenBracketToken, SyntaxKind.CloseBracketToken); return finishNode(node); } function parseParenthesizedType(): ParenthesizedTypeNode { const node = createNode(SyntaxKind.ParenthesizedType); parseExpected(SyntaxKind.OpenParenToken); node.type = parseType(); parseExpected(SyntaxKind.CloseParenToken); return finishNode(node); } function parseFunctionOrConstructorType(kind: SyntaxKind): FunctionOrConstructorTypeNode { const node = createNodeWithJSDoc(kind); if (kind === SyntaxKind.ConstructorType) { parseExpected(SyntaxKind.NewKeyword); } fillSignature(SyntaxKind.EqualsGreaterThanToken, SignatureFlags.Type, node); return finishNode(node); } function parseKeywordAndNoDot(): TypeNode | undefined { const node = parseTokenNode(); return token() === SyntaxKind.DotToken ? undefined : node; } function parseLiteralTypeNode(negative?: boolean): LiteralTypeNode { const node = createNode(SyntaxKind.LiteralType) as LiteralTypeNode; let unaryMinusExpression: PrefixUnaryExpression; if (negative) { unaryMinusExpression = createNode(SyntaxKind.PrefixUnaryExpression) as PrefixUnaryExpression; unaryMinusExpression.operator = SyntaxKind.MinusToken; nextToken(); } let expression: BooleanLiteral | LiteralExpression | PrefixUnaryExpression = token() === SyntaxKind.TrueKeyword || token() === SyntaxKind.FalseKeyword ? parseTokenNode() : parseLiteralLikeNode(token()) as LiteralExpression; if (negative) { unaryMinusExpression.operand = expression; finishNode(unaryMinusExpression); expression = unaryMinusExpression; } node.literal = expression; return finishNode(node); } function nextTokenIsNumericLiteral() { return nextToken() === SyntaxKind.NumericLiteral; } function parseNonArrayType(): TypeNode { switch (token()) { case SyntaxKind.AnyKeyword: case SyntaxKind.StringKeyword: case SyntaxKind.NumberKeyword: case SyntaxKind.SymbolKeyword: case SyntaxKind.BooleanKeyword: case SyntaxKind.UndefinedKeyword: case SyntaxKind.NeverKeyword: case SyntaxKind.ObjectKeyword: // If these are followed by a dot, then parse these out as a dotted type reference instead. return tryParse(parseKeywordAndNoDot) || parseTypeReference(); case SyntaxKind.AsteriskToken: return parseJSDocAllType(); case SyntaxKind.QuestionToken: return parseJSDocUnknownOrNullableType(); case SyntaxKind.FunctionKeyword: return parseJSDocFunctionType(); case SyntaxKind.ExclamationToken: return parseJSDocNodeWithType(SyntaxKind.JSDocNonNullableType); case SyntaxKind.NoSubstitutionTemplateLiteral: case SyntaxKind.StringLiteral: case SyntaxKind.NumericLiteral: case SyntaxKind.TrueKeyword: case SyntaxKind.FalseKeyword: return parseLiteralTypeNode(); case SyntaxKind.MinusToken: return lookAhead(nextTokenIsNumericLiteral) ? parseLiteralTypeNode(/*negative*/ true) : parseTypeReference(); case SyntaxKind.VoidKeyword: case SyntaxKind.NullKeyword: return parseTokenNode(); case SyntaxKind.ThisKeyword: { const thisKeyword = parseThisTypeNode(); if (token() === SyntaxKind.IsKeyword && !scanner.hasPrecedingLineBreak()) { return parseThisTypePredicate(thisKeyword); } else { return thisKeyword; } } case SyntaxKind.TypeOfKeyword: return parseTypeQuery(); case SyntaxKind.OpenBraceToken: return lookAhead(isStartOfMappedType) ? parseMappedType() : parseTypeLiteral(); case SyntaxKind.OpenBracketToken: return parseTupleType(); case SyntaxKind.OpenParenToken: return parseParenthesizedType(); default: return parseTypeReference(); } } function isStartOfType(inStartOfParameter?: boolean): boolean { switch (token()) { case SyntaxKind.AnyKeyword: case SyntaxKind.StringKeyword: case SyntaxKind.NumberKeyword: case SyntaxKind.BooleanKeyword: case SyntaxKind.SymbolKeyword: case SyntaxKind.UniqueKeyword: case SyntaxKind.VoidKeyword: case SyntaxKind.UndefinedKeyword: case SyntaxKind.NullKeyword: case SyntaxKind.ThisKeyword: case SyntaxKind.TypeOfKeyword: case SyntaxKind.NeverKeyword: case SyntaxKind.OpenBraceToken: case SyntaxKind.OpenBracketToken: case SyntaxKind.LessThanToken: case SyntaxKind.BarToken: case SyntaxKind.AmpersandToken: case SyntaxKind.NewKeyword: case SyntaxKind.StringLiteral: case SyntaxKind.NumericLiteral: case SyntaxKind.TrueKeyword: case SyntaxKind.FalseKeyword: case SyntaxKind.ObjectKeyword: case SyntaxKind.AsteriskToken: case SyntaxKind.QuestionToken: case SyntaxKind.ExclamationToken: case SyntaxKind.DotDotDotToken: case SyntaxKind.InferKeyword: return true; case SyntaxKind.MinusToken: return !inStartOfParameter && lookAhead(nextTokenIsNumericLiteral); case SyntaxKind.OpenParenToken: // Only consider '(' the start of a type if followed by ')', '...', an identifier, a modifier, // or something that starts a type. We don't want to consider things like '(1)' a type. return !inStartOfParameter && lookAhead(isStartOfParenthesizedOrFunctionType); default: return isIdentifier(); } } function isStartOfParenthesizedOrFunctionType() { nextToken(); return token() === SyntaxKind.CloseParenToken || isStartOfParameter() || isStartOfType(); } function parsePostfixTypeOrHigher(): TypeNode { let type = parseNonArrayType(); while (!scanner.hasPrecedingLineBreak()) { switch (token()) { case SyntaxKind.EqualsToken: // only parse postfix = inside jsdoc, because it's ambiguous elsewhere if (!(contextFlags & NodeFlags.JSDoc)) { return type; } type = createJSDocPostfixType(SyntaxKind.JSDocOptionalType, type); break; case SyntaxKind.ExclamationToken: type = createJSDocPostfixType(SyntaxKind.JSDocNonNullableType, type); break; case SyntaxKind.QuestionToken: // If not in JSDoc and next token is start of a type we have a conditional type if (!(contextFlags & NodeFlags.JSDoc) && lookAhead(nextTokenIsStartOfType)) { return type; } type = createJSDocPostfixType(SyntaxKind.JSDocNullableType, type); break; case SyntaxKind.OpenBracketToken: parseExpected(SyntaxKind.OpenBracketToken); if (isStartOfType()) { const node = createNode(SyntaxKind.IndexedAccessType, type.pos) as IndexedAccessTypeNode; node.objectType = type; node.indexType = parseType(); parseExpected(SyntaxKind.CloseBracketToken); type = finishNode(node); } else { const node = createNode(SyntaxKind.ArrayType, type.pos) as ArrayTypeNode; node.elementType = type; parseExpected(SyntaxKind.CloseBracketToken); type = finishNode(node); } break; default: return type; } } return type; } function createJSDocPostfixType(kind: SyntaxKind, type: TypeNode) { nextToken(); const postfix = createNode(kind, type.pos) as JSDocOptionalType | JSDocNonNullableType | JSDocNullableType; postfix.type = type; return finishNode(postfix); } function parseTypeOperator(operator: SyntaxKind.KeyOfKeyword | SyntaxKind.UniqueKeyword) { const node = createNode(SyntaxKind.TypeOperator); parseExpected(operator); node.operator = operator; node.type = parseTypeOperatorOrHigher(); return finishNode(node); } function parseInferType(): InferTypeNode { const node = createNode(SyntaxKind.InferType); parseExpected(SyntaxKind.InferKeyword); const typeParameter = createNode(SyntaxKind.TypeParameter); typeParameter.name = parseIdentifier(); node.typeParameter = finishNode(typeParameter); return finishNode(node); } function parseTypeOperatorOrHigher(): TypeNode { const operator = token(); switch (operator) { case SyntaxKind.KeyOfKeyword: case SyntaxKind.UniqueKeyword: return parseTypeOperator(operator); case SyntaxKind.InferKeyword: return parseInferType(); case SyntaxKind.DotDotDotToken: { const result = createNode(SyntaxKind.JSDocVariadicType) as JSDocVariadicType; nextToken(); result.type = parsePostfixTypeOrHigher(); return finishNode(result); } } return parsePostfixTypeOrHigher(); } function parseUnionOrIntersectionType(kind: SyntaxKind.UnionType | SyntaxKind.IntersectionType, parseConstituentType: () => TypeNode, operator: SyntaxKind.BarToken | SyntaxKind.AmpersandToken): TypeNode { parseOptional(operator); let type = parseConstituentType(); if (token() === operator) { const types = [type]; while (parseOptional(operator)) { types.push(parseConstituentType()); } const node = createNode(kind, type.pos); node.types = createNodeArray(types, type.pos); type = finishNode(node); } return type; } function parseIntersectionTypeOrHigher(): TypeNode { return parseUnionOrIntersectionType(SyntaxKind.IntersectionType, parseTypeOperatorOrHigher, SyntaxKind.AmpersandToken); } function parseUnionTypeOrHigher(): TypeNode { return parseUnionOrIntersectionType(SyntaxKind.UnionType, parseIntersectionTypeOrHigher, SyntaxKind.BarToken); } function isStartOfFunctionType(): boolean { if (token() === SyntaxKind.LessThanToken) { return true; } return token() === SyntaxKind.OpenParenToken && lookAhead(isUnambiguouslyStartOfFunctionType); } function skipParameterStart(): boolean { if (isModifierKind(token())) { // Skip modifiers parseModifiers(); } if (isIdentifier() || token() === SyntaxKind.ThisKeyword) { nextToken(); return true; } if (token() === SyntaxKind.OpenBracketToken || token() === SyntaxKind.OpenBraceToken) { // Return true if we can parse an array or object binding pattern with no errors const previousErrorCount = parseDiagnostics.length; parseIdentifierOrPattern(); return previousErrorCount === parseDiagnostics.length; } return false; } function isUnambiguouslyStartOfFunctionType() { nextToken(); if (token() === SyntaxKind.CloseParenToken || token() === SyntaxKind.DotDotDotToken) { // ( ) // ( ... return true; } if (skipParameterStart()) { // We successfully skipped modifiers (if any) and an identifier or binding pattern, // now see if we have something that indicates a parameter declaration if (token() === SyntaxKind.ColonToken || token() === SyntaxKind.CommaToken || token() === SyntaxKind.QuestionToken || token() === SyntaxKind.EqualsToken) { // ( xxx : // ( xxx , // ( xxx ? // ( xxx = return true; } if (token() === SyntaxKind.CloseParenToken) { nextToken(); if (token() === SyntaxKind.EqualsGreaterThanToken) { // ( xxx ) => return true; } } } return false; } function parseTypeOrTypePredicate(): TypeNode { const typePredicateVariable = isIdentifier() && tryParse(parseTypePredicatePrefix); const type = parseType(); if (typePredicateVariable) { const node = createNode(SyntaxKind.TypePredicate, typePredicateVariable.pos); node.parameterName = typePredicateVariable; node.type = type; return finishNode(node); } else { return type; } } function parseTypePredicatePrefix() { const id = parseIdentifier(); if (token() === SyntaxKind.IsKeyword && !scanner.hasPrecedingLineBreak()) { nextToken(); return id; } } function parseType(): TypeNode { // The rules about 'yield' only apply to actual code/expression contexts. They don't // apply to 'type' contexts. So we disable these parameters here before moving on. return doOutsideOfContext(NodeFlags.TypeExcludesFlags, parseTypeWorker); } function parseTypeWorker(noConditionalTypes?: boolean): TypeNode { if (isStartOfFunctionType()) { return parseFunctionOrConstructorType(SyntaxKind.FunctionType); } if (token() === SyntaxKind.NewKeyword) { return parseFunctionOrConstructorType(SyntaxKind.ConstructorType); } const type = parseUnionTypeOrHigher(); if (!noConditionalTypes && !scanner.hasPrecedingLineBreak() && parseOptional(SyntaxKind.ExtendsKeyword)) { const node = createNode(SyntaxKind.ConditionalType, type.pos); node.checkType = type; // The type following 'extends' is not permitted to be another conditional type node.extendsType = parseTypeWorker(/*noConditionalTypes*/ true); parseExpected(SyntaxKind.QuestionToken); node.trueType = parseTypeWorker(); parseExpected(SyntaxKind.ColonToken); node.falseType = parseTypeWorker(); return finishNode(node); } return type; } function parseTypeAnnotation(): TypeNode { return parseOptional(SyntaxKind.ColonToken) ? parseType() : undefined; } // EXPRESSIONS function isStartOfLeftHandSideExpression(): boolean { switch (token()) { case SyntaxKind.ThisKeyword: case SyntaxKind.SuperKeyword: case SyntaxKind.NullKeyword: case SyntaxKind.TrueKeyword: case SyntaxKind.FalseKeyword: case SyntaxKind.NumericLiteral: case SyntaxKind.StringLiteral: case SyntaxKind.NoSubstitutionTemplateLiteral: case SyntaxKind.TemplateHead: case SyntaxKind.OpenParenToken: case SyntaxKind.OpenBracketToken: case SyntaxKind.OpenBraceToken: case SyntaxKind.FunctionKeyword: case SyntaxKind.ClassKeyword: case SyntaxKind.NewKeyword: case SyntaxKind.SlashToken: case SyntaxKind.SlashEqualsToken: case SyntaxKind.Identifier: return true; case SyntaxKind.ImportKeyword: return lookAhead(nextTokenIsOpenParenOrLessThan); default: return isIdentifier(); } } function isStartOfExpression(): boolean { if (isStartOfLeftHandSideExpression()) { return true; } switch (token()) { case SyntaxKind.PlusToken: case SyntaxKind.MinusToken: case SyntaxKind.TildeToken: case SyntaxKind.ExclamationToken: case SyntaxKind.DeleteKeyword: case SyntaxKind.TypeOfKeyword: case SyntaxKind.VoidKeyword: case SyntaxKind.PlusPlusToken: case SyntaxKind.MinusMinusToken: case SyntaxKind.LessThanToken: case SyntaxKind.AwaitKeyword: case SyntaxKind.YieldKeyword: // Yield/await always starts an expression. Either it is an identifier (in which case // it is definitely an expression). Or it's a keyword (either because we're in // a generator or async function, or in strict mode (or both)) and it started a yield or await expression. return true; default: // Error tolerance. If we see the start of some binary operator, we consider // that the start of an expression. That way we'll parse out a missing identifier, // give a good message about an identifier being missing, and then consume the // rest of the binary expression. if (isBinaryOperator()) { return true; } return isIdentifier(); } } function isStartOfExpressionStatement(): boolean { // As per the grammar, none of '{' or 'function' or 'class' can start an expression statement. return token() !== SyntaxKind.OpenBraceToken && token() !== SyntaxKind.FunctionKeyword && token() !== SyntaxKind.ClassKeyword && token() !== SyntaxKind.AtToken && isStartOfExpression(); } function parseExpression(): Expression { // Expression[in]: // AssignmentExpression[in] // Expression[in] , AssignmentExpression[in] // clear the decorator context when parsing Expression, as it should be unambiguous when parsing a decorator const saveDecoratorContext = inDecoratorContext(); if (saveDecoratorContext) { setDecoratorContext(/*val*/ false); } let expr = parseAssignmentExpressionOrHigher(); let operatorToken: BinaryOperatorToken; while ((operatorToken = parseOptionalToken(SyntaxKind.CommaToken))) { expr = makeBinaryExpression(expr, operatorToken, parseAssignmentExpressionOrHigher()); } if (saveDecoratorContext) { setDecoratorContext(/*val*/ true); } return expr; } function parseInitializer(): Expression | undefined { return parseOptional(SyntaxKind.EqualsToken) ? parseAssignmentExpressionOrHigher() : undefined; } function parseAssignmentExpressionOrHigher(): Expression { // AssignmentExpression[in,yield]: // 1) ConditionalExpression[?in,?yield] // 2) LeftHandSideExpression = AssignmentExpression[?in,?yield] // 3) LeftHandSideExpression AssignmentOperator AssignmentExpression[?in,?yield] // 4) ArrowFunctionExpression[?in,?yield] // 5) AsyncArrowFunctionExpression[in,yield,await] // 6) [+Yield] YieldExpression[?In] // // Note: for ease of implementation we treat productions '2' and '3' as the same thing. // (i.e. they're both BinaryExpressions with an assignment operator in it). // First, do the simple check if we have a YieldExpression (production '6'). if (isYieldExpression()) { return parseYieldExpression(); } // Then, check if we have an arrow function (production '4' and '5') that starts with a parenthesized // parameter list or is an async arrow function. // AsyncArrowFunctionExpression: // 1) async[no LineTerminator here]AsyncArrowBindingIdentifier[?Yield][no LineTerminator here]=>AsyncConciseBody[?In] // 2) CoverCallExpressionAndAsyncArrowHead[?Yield, ?Await][no LineTerminator here]=>AsyncConciseBody[?In] // Production (1) of AsyncArrowFunctionExpression is parsed in "tryParseAsyncSimpleArrowFunctionExpression". // And production (2) is parsed in "tryParseParenthesizedArrowFunctionExpression". // // If we do successfully parse arrow-function, we must *not* recurse for productions 1, 2 or 3. An ArrowFunction is // not a LeftHandSideExpression, nor does it start a ConditionalExpression. So we are done // with AssignmentExpression if we see one. const arrowExpression = tryParseParenthesizedArrowFunctionExpression() || tryParseAsyncSimpleArrowFunctionExpression(); if (arrowExpression) { return arrowExpression; } // Now try to see if we're in production '1', '2' or '3'. A conditional expression can // start with a LogicalOrExpression, while the assignment productions can only start with // LeftHandSideExpressions. // // So, first, we try to just parse out a BinaryExpression. If we get something that is a // LeftHandSide or higher, then we can try to parse out the assignment expression part. // Otherwise, we try to parse out the conditional expression bit. We want to allow any // binary expression here, so we pass in the 'lowest' precedence here so that it matches // and consumes anything. const expr = parseBinaryExpressionOrHigher(/*precedence*/ 0); // To avoid a look-ahead, we did not handle the case of an arrow function with a single un-parenthesized // parameter ('x => ...') above. We handle it here by checking if the parsed expression was a single // identifier and the current token is an arrow. if (expr.kind === SyntaxKind.Identifier && token() === SyntaxKind.EqualsGreaterThanToken) { return parseSimpleArrowFunctionExpression(expr); } // Now see if we might be in cases '2' or '3'. // If the expression was a LHS expression, and we have an assignment operator, then // we're in '2' or '3'. Consume the assignment and return. // // Note: we call reScanGreaterToken so that we get an appropriately merged token // for cases like `> > =` becoming `>>=` if (isLeftHandSideExpression(expr) && isAssignmentOperator(reScanGreaterToken())) { return makeBinaryExpression(expr, parseTokenNode(), parseAssignmentExpressionOrHigher()); } // It wasn't an assignment or a lambda. This is a conditional expression: return parseConditionalExpressionRest(expr); } function isYieldExpression(): boolean { if (token() === SyntaxKind.YieldKeyword) { // If we have a 'yield' keyword, and this is a context where yield expressions are // allowed, then definitely parse out a yield expression. if (inYieldContext()) { return true; } // We're in a context where 'yield expr' is not allowed. However, if we can // definitely tell that the user was trying to parse a 'yield expr' and not // just a normal expr that start with a 'yield' identifier, then parse out // a 'yield expr'. We can then report an error later that they are only // allowed in generator expressions. // // for example, if we see 'yield(foo)', then we'll have to treat that as an // invocation expression of something called 'yield'. However, if we have // 'yield foo' then that is not legal as a normal expression, so we can // definitely recognize this as a yield expression. // // for now we just check if the next token is an identifier. More heuristics // can be added here later as necessary. We just need to make sure that we // don't accidentally consume something legal. return lookAhead(nextTokenIsIdentifierOrKeywordOrLiteralOnSameLine); } return false; } function nextTokenIsIdentifierOnSameLine() { nextToken(); return !scanner.hasPrecedingLineBreak() && isIdentifier(); } function parseYieldExpression(): YieldExpression { const node = createNode(SyntaxKind.YieldExpression); // YieldExpression[In] : // yield // yield [no LineTerminator here] [Lexical goal InputElementRegExp]AssignmentExpression[?In, Yield] // yield [no LineTerminator here] * [Lexical goal InputElementRegExp]AssignmentExpression[?In, Yield] nextToken(); if (!scanner.hasPrecedingLineBreak() && (token() === SyntaxKind.AsteriskToken || isStartOfExpression())) { node.asteriskToken = parseOptionalToken(SyntaxKind.AsteriskToken); node.expression = parseAssignmentExpressionOrHigher(); return finishNode(node); } else { // if the next token is not on the same line as yield. or we don't have an '*' or // the start of an expression, then this is just a simple "yield" expression. return finishNode(node); } } function parseSimpleArrowFunctionExpression(identifier: Identifier, asyncModifier?: NodeArray): ArrowFunction { Debug.assert(token() === SyntaxKind.EqualsGreaterThanToken, "parseSimpleArrowFunctionExpression should only have been called if we had a =>"); let node: ArrowFunction; if (asyncModifier) { node = createNode(SyntaxKind.ArrowFunction, asyncModifier.pos); node.modifiers = asyncModifier; } else { node = createNode(SyntaxKind.ArrowFunction, identifier.pos); } const parameter = createNode(SyntaxKind.Parameter, identifier.pos); parameter.name = identifier; finishNode(parameter); node.parameters = createNodeArray([parameter], parameter.pos, parameter.end); node.equalsGreaterThanToken = parseExpectedToken(SyntaxKind.EqualsGreaterThanToken); node.body = parseArrowFunctionExpressionBody(/*isAsync*/ !!asyncModifier); return addJSDocComment(finishNode(node)); } function tryParseParenthesizedArrowFunctionExpression(): Expression | undefined { const triState = isParenthesizedArrowFunctionExpression(); if (triState === Tristate.False) { // It's definitely not a parenthesized arrow function expression. return undefined; } // If we definitely have an arrow function, then we can just parse one, not requiring a // following => or { token. Otherwise, we *might* have an arrow function. Try to parse // it out, but don't allow any ambiguity, and return 'undefined' if this could be an // expression instead. const arrowFunction = triState === Tristate.True ? parseParenthesizedArrowFunctionExpressionHead(/*allowAmbiguity*/ true) : tryParse(parsePossibleParenthesizedArrowFunctionExpressionHead); if (!arrowFunction) { // Didn't appear to actually be a parenthesized arrow function. Just bail out. return undefined; } const isAsync = hasModifier(arrowFunction, ModifierFlags.Async); // If we have an arrow, then try to parse the body. Even if not, try to parse if we // have an opening brace, just in case we're in an error state. const lastToken = token(); arrowFunction.equalsGreaterThanToken = parseExpectedToken(SyntaxKind.EqualsGreaterThanToken); arrowFunction.body = (lastToken === SyntaxKind.EqualsGreaterThanToken || lastToken === SyntaxKind.OpenBraceToken) ? parseArrowFunctionExpressionBody(isAsync) : parseIdentifier(); return finishNode(arrowFunction); } // True -> We definitely expect a parenthesized arrow function here. // False -> There *cannot* be a parenthesized arrow function here. // Unknown -> There *might* be a parenthesized arrow function here. // Speculatively look ahead to be sure, and rollback if not. function isParenthesizedArrowFunctionExpression(): Tristate { if (token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.LessThanToken || token() === SyntaxKind.AsyncKeyword) { return lookAhead(isParenthesizedArrowFunctionExpressionWorker); } if (token() === SyntaxKind.EqualsGreaterThanToken) { // ERROR RECOVERY TWEAK: // If we see a standalone => try to parse it as an arrow function expression as that's // likely what the user intended to write. return Tristate.True; } // Definitely not a parenthesized arrow function. return Tristate.False; } function isParenthesizedArrowFunctionExpressionWorker() { if (token() === SyntaxKind.AsyncKeyword) { nextToken(); if (scanner.hasPrecedingLineBreak()) { return Tristate.False; } if (token() !== SyntaxKind.OpenParenToken && token() !== SyntaxKind.LessThanToken) { return Tristate.False; } } const first = token(); const second = nextToken(); if (first === SyntaxKind.OpenParenToken) { if (second === SyntaxKind.CloseParenToken) { // Simple cases: "() =>", "(): ", and "() {". // This is an arrow function with no parameters. // The last one is not actually an arrow function, // but this is probably what the user intended. const third = nextToken(); switch (third) { case SyntaxKind.EqualsGreaterThanToken: case SyntaxKind.ColonToken: case SyntaxKind.OpenBraceToken: return Tristate.True; default: return Tristate.False; } } // If encounter "([" or "({", this could be the start of a binding pattern. // Examples: // ([ x ]) => { } // ({ x }) => { } // ([ x ]) // ({ x }) if (second === SyntaxKind.OpenBracketToken || second === SyntaxKind.OpenBraceToken) { return Tristate.Unknown; } // Simple case: "(..." // This is an arrow function with a rest parameter. if (second === SyntaxKind.DotDotDotToken) { return Tristate.True; } // Check for "(xxx yyy", where xxx is a modifier and yyy is an identifier. This // isn't actually allowed, but we want to treat it as a lambda so we can provide // a good error message. if (isModifierKind(second) && second !== SyntaxKind.AsyncKeyword && lookAhead(nextTokenIsIdentifier)) { return Tristate.True; } // If we had "(" followed by something that's not an identifier, // then this definitely doesn't look like a lambda. if (!isIdentifier()) { return Tristate.False; } switch (nextToken()) { case SyntaxKind.ColonToken: // If we have something like "(a:", then we must have a // type-annotated parameter in an arrow function expression. return Tristate.True; case SyntaxKind.QuestionToken: nextToken(); // If we have "(a?:" or "(a?," or "(a?=" or "(a?)" then it is definitely a lambda. if (token() === SyntaxKind.ColonToken || token() === SyntaxKind.CommaToken || token() === SyntaxKind.EqualsToken || token() === SyntaxKind.CloseParenToken) { return Tristate.True; } // Otherwise it is definitely not a lambda. return Tristate.False; case SyntaxKind.CommaToken: case SyntaxKind.EqualsToken: case SyntaxKind.CloseParenToken: // If we have "(a," or "(a=" or "(a)" this *could* be an arrow function return Tristate.Unknown; } // It is definitely not an arrow function return Tristate.False; } else { Debug.assert(first === SyntaxKind.LessThanToken); // If we have "<" not followed by an identifier, // then this definitely is not an arrow function. if (!isIdentifier()) { return Tristate.False; } // JSX overrides if (sourceFile.languageVariant === LanguageVariant.JSX) { const isArrowFunctionInJsx = lookAhead(() => { const third = nextToken(); if (third === SyntaxKind.ExtendsKeyword) { const fourth = nextToken(); switch (fourth) { case SyntaxKind.EqualsToken: case SyntaxKind.GreaterThanToken: return false; default: return true; } } else if (third === SyntaxKind.CommaToken) { return true; } return false; }); if (isArrowFunctionInJsx) { return Tristate.True; } return Tristate.False; } // This *could* be a parenthesized arrow function. return Tristate.Unknown; } } function parsePossibleParenthesizedArrowFunctionExpressionHead(): ArrowFunction { return parseParenthesizedArrowFunctionExpressionHead(/*allowAmbiguity*/ false); } function tryParseAsyncSimpleArrowFunctionExpression(): ArrowFunction | undefined { // We do a check here so that we won't be doing unnecessarily call to "lookAhead" if (token() === SyntaxKind.AsyncKeyword) { if (lookAhead(isUnParenthesizedAsyncArrowFunctionWorker) === Tristate.True) { const asyncModifier = parseModifiersForArrowFunction(); const expr = parseBinaryExpressionOrHigher(/*precedence*/ 0); return parseSimpleArrowFunctionExpression(expr, asyncModifier); } } return undefined; } function isUnParenthesizedAsyncArrowFunctionWorker(): Tristate { // AsyncArrowFunctionExpression: // 1) async[no LineTerminator here]AsyncArrowBindingIdentifier[?Yield][no LineTerminator here]=>AsyncConciseBody[?In] // 2) CoverCallExpressionAndAsyncArrowHead[?Yield, ?Await][no LineTerminator here]=>AsyncConciseBody[?In] if (token() === SyntaxKind.AsyncKeyword) { nextToken(); // If the "async" is followed by "=>" token then it is not a begining of an async arrow-function // but instead a simple arrow-function which will be parsed inside "parseAssignmentExpressionOrHigher" if (scanner.hasPrecedingLineBreak() || token() === SyntaxKind.EqualsGreaterThanToken) { return Tristate.False; } // Check for un-parenthesized AsyncArrowFunction const expr = parseBinaryExpressionOrHigher(/*precedence*/ 0); if (!scanner.hasPrecedingLineBreak() && expr.kind === SyntaxKind.Identifier && token() === SyntaxKind.EqualsGreaterThanToken) { return Tristate.True; } } return Tristate.False; } function parseParenthesizedArrowFunctionExpressionHead(allowAmbiguity: boolean): ArrowFunction { const node = createNodeWithJSDoc(SyntaxKind.ArrowFunction); node.modifiers = parseModifiersForArrowFunction(); const isAsync = hasModifier(node, ModifierFlags.Async) ? SignatureFlags.Await : SignatureFlags.None; // Arrow functions are never generators. // // If we're speculatively parsing a signature for a parenthesized arrow function, then // we have to have a complete parameter list. Otherwise we might see something like // a => (b => c) // And think that "(b =>" was actually a parenthesized arrow function with a missing // close paren. fillSignature(SyntaxKind.ColonToken, isAsync | (allowAmbiguity ? SignatureFlags.None : SignatureFlags.RequireCompleteParameterList), node); // If we couldn't get parameters, we definitely could not parse out an arrow function. if (!node.parameters) { return undefined; } // Parsing a signature isn't enough. // Parenthesized arrow signatures often look like other valid expressions. // For instance: // - "(x = 10)" is an assignment expression parsed as a signature with a default parameter value. // - "(x,y)" is a comma expression parsed as a signature with two parameters. // - "a ? (b): c" will have "(b):" parsed as a signature with a return type annotation. // // So we need just a bit of lookahead to ensure that it can only be a signature. if (!allowAmbiguity && token() !== SyntaxKind.EqualsGreaterThanToken && token() !== SyntaxKind.OpenBraceToken) { // Returning undefined here will cause our caller to rewind to where we started from. return undefined; } return node; } function parseArrowFunctionExpressionBody(isAsync: boolean): Block | Expression { if (token() === SyntaxKind.OpenBraceToken) { return parseFunctionBlock(isAsync ? SignatureFlags.Await : SignatureFlags.None); } if (token() !== SyntaxKind.SemicolonToken && token() !== SyntaxKind.FunctionKeyword && token() !== SyntaxKind.ClassKeyword && isStartOfStatement() && !isStartOfExpressionStatement()) { // Check if we got a plain statement (i.e. no expression-statements, no function/class expressions/declarations) // // Here we try to recover from a potential error situation in the case where the // user meant to supply a block. For example, if the user wrote: // // a => // let v = 0; // } // // they may be missing an open brace. Check to see if that's the case so we can // try to recover better. If we don't do this, then the next close curly we see may end // up preemptively closing the containing construct. // // Note: even when 'IgnoreMissingOpenBrace' is passed, parseBody will still error. return parseFunctionBlock(SignatureFlags.IgnoreMissingOpenBrace | (isAsync ? SignatureFlags.Await : SignatureFlags.None)); } return isAsync ? doInAwaitContext(parseAssignmentExpressionOrHigher) : doOutsideOfAwaitContext(parseAssignmentExpressionOrHigher); } function parseConditionalExpressionRest(leftOperand: Expression): Expression { // Note: we are passed in an expression which was produced from parseBinaryExpressionOrHigher. const questionToken = parseOptionalToken(SyntaxKind.QuestionToken); if (!questionToken) { return leftOperand; } // Note: we explicitly 'allowIn' in the whenTrue part of the condition expression, and // we do not that for the 'whenFalse' part. const node = createNode(SyntaxKind.ConditionalExpression, leftOperand.pos); node.condition = leftOperand; node.questionToken = questionToken; node.whenTrue = doOutsideOfContext(disallowInAndDecoratorContext, parseAssignmentExpressionOrHigher); node.colonToken = parseExpectedToken(SyntaxKind.ColonToken); node.whenFalse = nodeIsPresent(node.colonToken) ? parseAssignmentExpressionOrHigher() : createMissingNode(SyntaxKind.Identifier, /*reportAtCurrentPosition*/ false, Diagnostics._0_expected, tokenToString(SyntaxKind.ColonToken)); return finishNode(node); } function parseBinaryExpressionOrHigher(precedence: number): Expression { const leftOperand = parseUnaryExpressionOrHigher(); return parseBinaryExpressionRest(precedence, leftOperand); } function isInOrOfKeyword(t: SyntaxKind) { return t === SyntaxKind.InKeyword || t === SyntaxKind.OfKeyword; } function parseBinaryExpressionRest(precedence: number, leftOperand: Expression): Expression { while (true) { // We either have a binary operator here, or we're finished. We call // reScanGreaterToken so that we merge token sequences like > and = into >= reScanGreaterToken(); const newPrecedence = getBinaryOperatorPrecedence(); // Check the precedence to see if we should "take" this operator // - For left associative operator (all operator but **), consume the operator, // recursively call the function below, and parse binaryExpression as a rightOperand // of the caller if the new precedence of the operator is greater then or equal to the current precedence. // For example: // a - b - c; // ^token; leftOperand = b. Return b to the caller as a rightOperand // a * b - c // ^token; leftOperand = b. Return b to the caller as a rightOperand // a - b * c; // ^token; leftOperand = b. Return b * c to the caller as a rightOperand // - For right associative operator (**), consume the operator, recursively call the function // and parse binaryExpression as a rightOperand of the caller if the new precedence of // the operator is strictly grater than the current precedence // For example: // a ** b ** c; // ^^token; leftOperand = b. Return b ** c to the caller as a rightOperand // a - b ** c; // ^^token; leftOperand = b. Return b ** c to the caller as a rightOperand // a ** b - c // ^token; leftOperand = b. Return b to the caller as a rightOperand const consumeCurrentOperator = token() === SyntaxKind.AsteriskAsteriskToken ? newPrecedence >= precedence : newPrecedence > precedence; if (!consumeCurrentOperator) { break; } if (token() === SyntaxKind.InKeyword && inDisallowInContext()) { break; } if (token() === SyntaxKind.AsKeyword) { // Make sure we *do* perform ASI for constructs like this: // var x = foo // as (Bar) // This should be parsed as an initialized variable, followed // by a function call to 'as' with the argument 'Bar' if (scanner.hasPrecedingLineBreak()) { break; } else { nextToken(); leftOperand = makeAsExpression(leftOperand, parseType()); } } else { leftOperand = makeBinaryExpression(leftOperand, parseTokenNode(), parseBinaryExpressionOrHigher(newPrecedence)); } } return leftOperand; } function isBinaryOperator() { if (inDisallowInContext() && token() === SyntaxKind.InKeyword) { return false; } return getBinaryOperatorPrecedence() > 0; } function getBinaryOperatorPrecedence(): number { switch (token()) { case SyntaxKind.BarBarToken: return 1; case SyntaxKind.AmpersandAmpersandToken: return 2; case SyntaxKind.BarToken: return 3; case SyntaxKind.CaretToken: return 4; case SyntaxKind.AmpersandToken: return 5; case SyntaxKind.EqualsEqualsToken: case SyntaxKind.ExclamationEqualsToken: case SyntaxKind.EqualsEqualsEqualsToken: case SyntaxKind.ExclamationEqualsEqualsToken: return 6; case SyntaxKind.LessThanToken: case SyntaxKind.GreaterThanToken: case SyntaxKind.LessThanEqualsToken: case SyntaxKind.GreaterThanEqualsToken: case SyntaxKind.InstanceOfKeyword: case SyntaxKind.InKeyword: case SyntaxKind.AsKeyword: return 7; case SyntaxKind.LessThanLessThanToken: case SyntaxKind.GreaterThanGreaterThanToken: case SyntaxKind.GreaterThanGreaterThanGreaterThanToken: return 8; case SyntaxKind.PlusToken: case SyntaxKind.MinusToken: return 9; case SyntaxKind.AsteriskToken: case SyntaxKind.SlashToken: case SyntaxKind.PercentToken: return 10; case SyntaxKind.AsteriskAsteriskToken: return 11; } // -1 is lower than all other precedences. Returning it will cause binary expression // parsing to stop. return -1; } function makeBinaryExpression(left: Expression, operatorToken: BinaryOperatorToken, right: Expression): BinaryExpression { const node = createNode(SyntaxKind.BinaryExpression, left.pos); node.left = left; node.operatorToken = operatorToken; node.right = right; return finishNode(node); } function makeAsExpression(left: Expression, right: TypeNode): AsExpression { const node = createNode(SyntaxKind.AsExpression, left.pos); node.expression = left; node.type = right; return finishNode(node); } function parsePrefixUnaryExpression() { const node = createNode(SyntaxKind.PrefixUnaryExpression); node.operator = token(); nextToken(); node.operand = parseSimpleUnaryExpression(); return finishNode(node); } function parseDeleteExpression() { const node = createNode(SyntaxKind.DeleteExpression); nextToken(); node.expression = parseSimpleUnaryExpression(); return finishNode(node); } function parseTypeOfExpression() { const node = createNode(SyntaxKind.TypeOfExpression); nextToken(); node.expression = parseSimpleUnaryExpression(); return finishNode(node); } function parseVoidExpression() { const node = createNode(SyntaxKind.VoidExpression); nextToken(); node.expression = parseSimpleUnaryExpression(); return finishNode(node); } function isAwaitExpression(): boolean { if (token() === SyntaxKind.AwaitKeyword) { if (inAwaitContext()) { return true; } // here we are using similar heuristics as 'isYieldExpression' return lookAhead(nextTokenIsIdentifierOrKeywordOrLiteralOnSameLine); } return false; } function parseAwaitExpression() { const node = createNode(SyntaxKind.AwaitExpression); nextToken(); node.expression = parseSimpleUnaryExpression(); return finishNode(node); } /** * Parse ES7 exponential expression and await expression * * ES7 ExponentiationExpression: * 1) UnaryExpression[?Yield] * 2) UpdateExpression[?Yield] ** ExponentiationExpression[?Yield] * */ function parseUnaryExpressionOrHigher(): UnaryExpression | BinaryExpression { /** * ES7 UpdateExpression: * 1) LeftHandSideExpression[?Yield] * 2) LeftHandSideExpression[?Yield][no LineTerminator here]++ * 3) LeftHandSideExpression[?Yield][no LineTerminator here]-- * 4) ++UnaryExpression[?Yield] * 5) --UnaryExpression[?Yield] */ if (isUpdateExpression()) { const updateExpression = parseUpdateExpression(); return token() === SyntaxKind.AsteriskAsteriskToken ? parseBinaryExpressionRest(getBinaryOperatorPrecedence(), updateExpression) : updateExpression; } /** * ES7 UnaryExpression: * 1) UpdateExpression[?yield] * 2) delete UpdateExpression[?yield] * 3) void UpdateExpression[?yield] * 4) typeof UpdateExpression[?yield] * 5) + UpdateExpression[?yield] * 6) - UpdateExpression[?yield] * 7) ~ UpdateExpression[?yield] * 8) ! UpdateExpression[?yield] */ const unaryOperator = token(); const simpleUnaryExpression = parseSimpleUnaryExpression(); if (token() === SyntaxKind.AsteriskAsteriskToken) { const start = skipTrivia(sourceText, simpleUnaryExpression.pos); if (simpleUnaryExpression.kind === SyntaxKind.TypeAssertionExpression) { parseErrorAtPosition(start, simpleUnaryExpression.end - start, Diagnostics.A_type_assertion_expression_is_not_allowed_in_the_left_hand_side_of_an_exponentiation_expression_Consider_enclosing_the_expression_in_parentheses); } else { parseErrorAtPosition(start, simpleUnaryExpression.end - start, Diagnostics.An_unary_expression_with_the_0_operator_is_not_allowed_in_the_left_hand_side_of_an_exponentiation_expression_Consider_enclosing_the_expression_in_parentheses, tokenToString(unaryOperator)); } } return simpleUnaryExpression; } /** * Parse ES7 simple-unary expression or higher: * * ES7 UnaryExpression: * 1) UpdateExpression[?yield] * 2) delete UnaryExpression[?yield] * 3) void UnaryExpression[?yield] * 4) typeof UnaryExpression[?yield] * 5) + UnaryExpression[?yield] * 6) - UnaryExpression[?yield] * 7) ~ UnaryExpression[?yield] * 8) ! UnaryExpression[?yield] * 9) [+Await] await UnaryExpression[?yield] */ function parseSimpleUnaryExpression(): UnaryExpression { switch (token()) { case SyntaxKind.PlusToken: case SyntaxKind.MinusToken: case SyntaxKind.TildeToken: case SyntaxKind.ExclamationToken: return parsePrefixUnaryExpression(); case SyntaxKind.DeleteKeyword: return parseDeleteExpression(); case SyntaxKind.TypeOfKeyword: return parseTypeOfExpression(); case SyntaxKind.VoidKeyword: return parseVoidExpression(); case SyntaxKind.LessThanToken: // This is modified UnaryExpression grammar in TypeScript // UnaryExpression (modified): // < type > UnaryExpression return parseTypeAssertion(); case SyntaxKind.AwaitKeyword: if (isAwaitExpression()) { return parseAwaitExpression(); } // falls through default: return parseUpdateExpression(); } } /** * Check if the current token can possibly be an ES7 increment expression. * * ES7 UpdateExpression: * LeftHandSideExpression[?Yield] * LeftHandSideExpression[?Yield][no LineTerminator here]++ * LeftHandSideExpression[?Yield][no LineTerminator here]-- * ++LeftHandSideExpression[?Yield] * --LeftHandSideExpression[?Yield] */ function isUpdateExpression(): boolean { // This function is called inside parseUnaryExpression to decide // whether to call parseSimpleUnaryExpression or call parseUpdateExpression directly switch (token()) { case SyntaxKind.PlusToken: case SyntaxKind.MinusToken: case SyntaxKind.TildeToken: case SyntaxKind.ExclamationToken: case SyntaxKind.DeleteKeyword: case SyntaxKind.TypeOfKeyword: case SyntaxKind.VoidKeyword: case SyntaxKind.AwaitKeyword: return false; case SyntaxKind.LessThanToken: // If we are not in JSX context, we are parsing TypeAssertion which is an UnaryExpression if (sourceFile.languageVariant !== LanguageVariant.JSX) { return false; } // We are in JSX context and the token is part of JSXElement. // falls through default: return true; } } /** * Parse ES7 UpdateExpression. UpdateExpression is used instead of ES6's PostFixExpression. * * ES7 UpdateExpression[yield]: * 1) LeftHandSideExpression[?yield] * 2) LeftHandSideExpression[?yield] [[no LineTerminator here]]++ * 3) LeftHandSideExpression[?yield] [[no LineTerminator here]]-- * 4) ++LeftHandSideExpression[?yield] * 5) --LeftHandSideExpression[?yield] * In TypeScript (2), (3) are parsed as PostfixUnaryExpression. (4), (5) are parsed as PrefixUnaryExpression */ function parseUpdateExpression(): UpdateExpression { if (token() === SyntaxKind.PlusPlusToken || token() === SyntaxKind.MinusMinusToken) { const node = createNode(SyntaxKind.PrefixUnaryExpression); node.operator = token(); nextToken(); node.operand = parseLeftHandSideExpressionOrHigher(); return finishNode(node); } else if (sourceFile.languageVariant === LanguageVariant.JSX && token() === SyntaxKind.LessThanToken && lookAhead(nextTokenIsIdentifierOrKeywordOrGreaterThan)) { // JSXElement is part of primaryExpression return parseJsxElementOrSelfClosingElementOrFragment(/*inExpressionContext*/ true); } const expression = parseLeftHandSideExpressionOrHigher(); Debug.assert(isLeftHandSideExpression(expression)); if ((token() === SyntaxKind.PlusPlusToken || token() === SyntaxKind.MinusMinusToken) && !scanner.hasPrecedingLineBreak()) { const node = createNode(SyntaxKind.PostfixUnaryExpression, expression.pos); node.operand = expression; node.operator = token(); nextToken(); return finishNode(node); } return expression; } function parseLeftHandSideExpressionOrHigher(): LeftHandSideExpression { // Original Ecma: // LeftHandSideExpression: See 11.2 // NewExpression // CallExpression // // Our simplification: // // LeftHandSideExpression: See 11.2 // MemberExpression // CallExpression // // See comment in parseMemberExpressionOrHigher on how we replaced NewExpression with // MemberExpression to make our lives easier. // // to best understand the below code, it's important to see how CallExpression expands // out into its own productions: // // CallExpression: // MemberExpression Arguments // CallExpression Arguments // CallExpression[Expression] // CallExpression.IdentifierName // import (AssignmentExpression) // super Arguments // super.IdentifierName // // Because of the recursion in these calls, we need to bottom out first. There are three // bottom out states we can run into: 1) We see 'super' which must start either of // the last two CallExpression productions. 2) We see 'import' which must start import call. // 3)we have a MemberExpression which either completes the LeftHandSideExpression, // or starts the beginning of the first four CallExpression productions. let expression: MemberExpression; if (token() === SyntaxKind.ImportKeyword && lookAhead(nextTokenIsOpenParenOrLessThan)) { // We don't want to eagerly consume all import keyword as import call expression so we look a head to find "(" // For example: // var foo3 = require("subfolder // import * as foo1 from "module-from-node // We want this import to be a statement rather than import call expression sourceFile.flags |= NodeFlags.PossiblyContainsDynamicImport; expression = parseTokenNode(); } else { expression = token() === SyntaxKind.SuperKeyword ? parseSuperExpression() : parseMemberExpressionOrHigher(); } // Now, we *may* be complete. However, we might have consumed the start of a // CallExpression. As such, we need to consume the rest of it here to be complete. return parseCallExpressionRest(expression); } function parseMemberExpressionOrHigher(): MemberExpression { // Note: to make our lives simpler, we decompose the NewExpression productions and // place ObjectCreationExpression and FunctionExpression into PrimaryExpression. // like so: // // PrimaryExpression : See 11.1 // this // Identifier // Literal // ArrayLiteral // ObjectLiteral // (Expression) // FunctionExpression // new MemberExpression Arguments? // // MemberExpression : See 11.2 // PrimaryExpression // MemberExpression[Expression] // MemberExpression.IdentifierName // // CallExpression : See 11.2 // MemberExpression // CallExpression Arguments // CallExpression[Expression] // CallExpression.IdentifierName // // Technically this is ambiguous. i.e. CallExpression defines: // // CallExpression: // CallExpression Arguments // // If you see: "new Foo()" // // Then that could be treated as a single ObjectCreationExpression, or it could be // treated as the invocation of "new Foo". We disambiguate that in code (to match // the original grammar) by making sure that if we see an ObjectCreationExpression // we always consume arguments if they are there. So we treat "new Foo()" as an // object creation only, and not at all as an invocation. Another way to think // about this is that for every "new" that we see, we will consume an argument list if // it is there as part of the *associated* object creation node. Any additional // argument lists we see, will become invocation expressions. // // Because there are no other places in the grammar now that refer to FunctionExpression // or ObjectCreationExpression, it is safe to push down into the PrimaryExpression // production. // // Because CallExpression and MemberExpression are left recursive, we need to bottom out // of the recursion immediately. So we parse out a primary expression to start with. const expression = parsePrimaryExpression(); return parseMemberExpressionRest(expression); } function parseSuperExpression(): MemberExpression { const expression = parseTokenNode(); if (token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.DotToken || token() === SyntaxKind.OpenBracketToken) { return expression; } // If we have seen "super" it must be followed by '(' or '.'. // If it wasn't then just try to parse out a '.' and report an error. const node = createNode(SyntaxKind.PropertyAccessExpression, expression.pos); node.expression = expression; parseExpectedToken(SyntaxKind.DotToken, Diagnostics.super_must_be_followed_by_an_argument_list_or_member_access); node.name = parseRightSideOfDot(/*allowIdentifierNames*/ true); return finishNode(node); } function tagNamesAreEquivalent(lhs: JsxTagNameExpression, rhs: JsxTagNameExpression): boolean { if (lhs.kind !== rhs.kind) { return false; } if (lhs.kind === SyntaxKind.Identifier) { return (lhs).escapedText === (rhs).escapedText; } if (lhs.kind === SyntaxKind.ThisKeyword) { return true; } // If we are at this statement then we must have PropertyAccessExpression and because tag name in Jsx element can only // take forms of JsxTagNameExpression which includes an identifier, "this" expression, or another propertyAccessExpression // it is safe to case the expression property as such. See parseJsxElementName for how we parse tag name in Jsx element return (lhs).name.escapedText === (rhs).name.escapedText && tagNamesAreEquivalent((lhs).expression as JsxTagNameExpression, (rhs).expression as JsxTagNameExpression); } function parseJsxElementOrSelfClosingElementOrFragment(inExpressionContext: boolean): JsxElement | JsxSelfClosingElement | JsxFragment { const opening = parseJsxOpeningOrSelfClosingElementOrOpeningFragment(inExpressionContext); let result: JsxElement | JsxSelfClosingElement | JsxFragment; if (opening.kind === SyntaxKind.JsxOpeningElement) { const node = createNode(SyntaxKind.JsxElement, opening.pos); node.openingElement = opening; node.children = parseJsxChildren(node.openingElement); node.closingElement = parseJsxClosingElement(inExpressionContext); if (!tagNamesAreEquivalent(node.openingElement.tagName, node.closingElement.tagName)) { parseErrorAtPosition(node.closingElement.pos, node.closingElement.end - node.closingElement.pos, Diagnostics.Expected_corresponding_JSX_closing_tag_for_0, getTextOfNodeFromSourceText(sourceText, node.openingElement.tagName)); } result = finishNode(node); } else if (opening.kind === SyntaxKind.JsxOpeningFragment) { const node = createNode(SyntaxKind.JsxFragment, opening.pos); node.openingFragment = opening; node.children = parseJsxChildren(node.openingFragment); node.closingFragment = parseJsxClosingFragment(inExpressionContext); result = finishNode(node); } else { Debug.assert(opening.kind === SyntaxKind.JsxSelfClosingElement); // Nothing else to do for self-closing elements result = opening; } // If the user writes the invalid code '
' in an expression context (i.e. not wrapped in // an enclosing tag), we'll naively try to parse ^ this as a 'less than' operator and the remainder of the tag // as garbage, which will cause the formatter to badly mangle the JSX. Perform a speculative parse of a JSX // element if we see a < token so that we can wrap it in a synthetic binary expression so the formatter // does less damage and we can report a better error. // Since JSX elements are invalid < operands anyway, this lookahead parse will only occur in error scenarios // of one sort or another. if (inExpressionContext && token() === SyntaxKind.LessThanToken) { const invalidElement = tryParse(() => parseJsxElementOrSelfClosingElementOrFragment(/*inExpressionContext*/ true)); if (invalidElement) { parseErrorAtCurrentToken(Diagnostics.JSX_expressions_must_have_one_parent_element); const badNode = createNode(SyntaxKind.BinaryExpression, result.pos); badNode.end = invalidElement.end; badNode.left = result; badNode.right = invalidElement; badNode.operatorToken = createMissingNode(SyntaxKind.CommaToken, /*reportAtCurrentPosition*/ false, /*diagnosticMessage*/ undefined); badNode.operatorToken.pos = badNode.operatorToken.end = badNode.right.pos; return badNode; } } return result; } function parseJsxText(): JsxText { const node = createNode(SyntaxKind.JsxText); node.containsOnlyWhiteSpaces = currentToken === SyntaxKind.JsxTextAllWhiteSpaces; currentToken = scanner.scanJsxToken(); return finishNode(node); } function parseJsxChild(): JsxChild { switch (token()) { case SyntaxKind.JsxText: case SyntaxKind.JsxTextAllWhiteSpaces: return parseJsxText(); case SyntaxKind.OpenBraceToken: return parseJsxExpression(/*inExpressionContext*/ false); case SyntaxKind.LessThanToken: return parseJsxElementOrSelfClosingElementOrFragment(/*inExpressionContext*/ false); } Debug.fail("Unknown JSX child kind " + token()); } function parseJsxChildren(openingTag: JsxOpeningElement | JsxOpeningFragment): NodeArray { const list = []; const listPos = getNodePos(); const saveParsingContext = parsingContext; parsingContext |= 1 << ParsingContext.JsxChildren; while (true) { currentToken = scanner.reScanJsxToken(); if (token() === SyntaxKind.LessThanSlashToken) { // Closing tag break; } else if (token() === SyntaxKind.EndOfFileToken) { // If we hit EOF, issue the error at the tag that lacks the closing element // rather than at the end of the file (which is useless) if (isJsxOpeningFragment(openingTag)) { parseErrorAtPosition(openingTag.pos, openingTag.end - openingTag.pos, Diagnostics.JSX_fragment_has_no_corresponding_closing_tag); } else { const openingTagName = openingTag.tagName; parseErrorAtPosition(openingTagName.pos, openingTagName.end - openingTagName.pos, Diagnostics.JSX_element_0_has_no_corresponding_closing_tag, getTextOfNodeFromSourceText(sourceText, openingTagName)); } break; } else if (token() === SyntaxKind.ConflictMarkerTrivia) { break; } const child = parseJsxChild(); if (child) { list.push(child); } } parsingContext = saveParsingContext; return createNodeArray(list, listPos); } function parseJsxAttributes(): JsxAttributes { const jsxAttributes = createNode(SyntaxKind.JsxAttributes); jsxAttributes.properties = parseList(ParsingContext.JsxAttributes, parseJsxAttribute); return finishNode(jsxAttributes); } function parseJsxOpeningOrSelfClosingElementOrOpeningFragment(inExpressionContext: boolean): JsxOpeningElement | JsxSelfClosingElement | JsxOpeningFragment { const fullStart = scanner.getStartPos(); parseExpected(SyntaxKind.LessThanToken); if (token() === SyntaxKind.GreaterThanToken) { parseExpected(SyntaxKind.GreaterThanToken); const node: JsxOpeningFragment = createNode(SyntaxKind.JsxOpeningFragment, fullStart); return finishNode(node); } const tagName = parseJsxElementName(); const attributes = parseJsxAttributes(); let node: JsxOpeningLikeElement; if (token() === SyntaxKind.GreaterThanToken) { // Closing tag, so scan the immediately-following text with the JSX scanning instead // of regular scanning to avoid treating illegal characters (e.g. '#') as immediate // scanning errors node = createNode(SyntaxKind.JsxOpeningElement, fullStart); scanJsxText(); } else { parseExpected(SyntaxKind.SlashToken); if (inExpressionContext) { parseExpected(SyntaxKind.GreaterThanToken); } else { parseExpected(SyntaxKind.GreaterThanToken, /*diagnostic*/ undefined, /*shouldAdvance*/ false); scanJsxText(); } node = createNode(SyntaxKind.JsxSelfClosingElement, fullStart); } node.tagName = tagName; node.attributes = attributes; return finishNode(node); } function parseJsxElementName(): JsxTagNameExpression { scanJsxIdentifier(); // JsxElement can have name in the form of // propertyAccessExpression // primaryExpression in the form of an identifier and "this" keyword // We can't just simply use parseLeftHandSideExpressionOrHigher because then we will start consider class,function etc as a keyword // We only want to consider "this" as a primaryExpression let expression: JsxTagNameExpression = token() === SyntaxKind.ThisKeyword ? parseTokenNode() : parseIdentifierName(); while (parseOptional(SyntaxKind.DotToken)) { const propertyAccess: PropertyAccessExpression = createNode(SyntaxKind.PropertyAccessExpression, expression.pos); propertyAccess.expression = expression; propertyAccess.name = parseRightSideOfDot(/*allowIdentifierNames*/ true); expression = finishNode(propertyAccess); } return expression; } function parseJsxExpression(inExpressionContext: boolean): JsxExpression { const node = createNode(SyntaxKind.JsxExpression); parseExpected(SyntaxKind.OpenBraceToken); if (token() !== SyntaxKind.CloseBraceToken) { node.dotDotDotToken = parseOptionalToken(SyntaxKind.DotDotDotToken); node.expression = parseAssignmentExpressionOrHigher(); } if (inExpressionContext) { parseExpected(SyntaxKind.CloseBraceToken); } else { parseExpected(SyntaxKind.CloseBraceToken, /*message*/ undefined, /*shouldAdvance*/ false); scanJsxText(); } return finishNode(node); } function parseJsxAttribute(): JsxAttribute | JsxSpreadAttribute { if (token() === SyntaxKind.OpenBraceToken) { return parseJsxSpreadAttribute(); } scanJsxIdentifier(); const node = createNode(SyntaxKind.JsxAttribute); node.name = parseIdentifierName(); if (token() === SyntaxKind.EqualsToken) { switch (scanJsxAttributeValue()) { case SyntaxKind.StringLiteral: node.initializer = parseLiteralNode(); break; default: node.initializer = parseJsxExpression(/*inExpressionContext*/ true); break; } } return finishNode(node); } function parseJsxSpreadAttribute(): JsxSpreadAttribute { const node = createNode(SyntaxKind.JsxSpreadAttribute); parseExpected(SyntaxKind.OpenBraceToken); parseExpected(SyntaxKind.DotDotDotToken); node.expression = parseExpression(); parseExpected(SyntaxKind.CloseBraceToken); return finishNode(node); } function parseJsxClosingElement(inExpressionContext: boolean): JsxClosingElement { const node = createNode(SyntaxKind.JsxClosingElement); parseExpected(SyntaxKind.LessThanSlashToken); node.tagName = parseJsxElementName(); if (inExpressionContext) { parseExpected(SyntaxKind.GreaterThanToken); } else { parseExpected(SyntaxKind.GreaterThanToken, /*diagnostic*/ undefined, /*shouldAdvance*/ false); scanJsxText(); } return finishNode(node); } function parseJsxClosingFragment(inExpressionContext: boolean): JsxClosingFragment { const node = createNode(SyntaxKind.JsxClosingFragment); parseExpected(SyntaxKind.LessThanSlashToken); if (tokenIsIdentifierOrKeyword(token())) { const unexpectedTagName = parseJsxElementName(); parseErrorAtPosition(unexpectedTagName.pos, unexpectedTagName.end - unexpectedTagName.pos, Diagnostics.Expected_corresponding_closing_tag_for_JSX_fragment); } if (inExpressionContext) { parseExpected(SyntaxKind.GreaterThanToken); } else { parseExpected(SyntaxKind.GreaterThanToken, /*diagnostic*/ undefined, /*shouldAdvance*/ false); scanJsxText(); } return finishNode(node); } function parseTypeAssertion(): TypeAssertion { const node = createNode(SyntaxKind.TypeAssertionExpression); parseExpected(SyntaxKind.LessThanToken); node.type = parseType(); parseExpected(SyntaxKind.GreaterThanToken); node.expression = parseSimpleUnaryExpression(); return finishNode(node); } function parseMemberExpressionRest(expression: LeftHandSideExpression): MemberExpression { while (true) { const dotToken = parseOptionalToken(SyntaxKind.DotToken); if (dotToken) { const propertyAccess = createNode(SyntaxKind.PropertyAccessExpression, expression.pos); propertyAccess.expression = expression; propertyAccess.name = parseRightSideOfDot(/*allowIdentifierNames*/ true); expression = finishNode(propertyAccess); continue; } if (token() === SyntaxKind.ExclamationToken && !scanner.hasPrecedingLineBreak()) { nextToken(); const nonNullExpression = createNode(SyntaxKind.NonNullExpression, expression.pos); nonNullExpression.expression = expression; expression = finishNode(nonNullExpression); continue; } // when in the [Decorator] context, we do not parse ElementAccess as it could be part of a ComputedPropertyName if (!inDecoratorContext() && parseOptional(SyntaxKind.OpenBracketToken)) { const indexedAccess = createNode(SyntaxKind.ElementAccessExpression, expression.pos); indexedAccess.expression = expression; // It's not uncommon for a user to write: "new Type[]". // Check for that common pattern and report a better error message. if (token() !== SyntaxKind.CloseBracketToken) { indexedAccess.argumentExpression = allowInAnd(parseExpression); if (indexedAccess.argumentExpression.kind === SyntaxKind.StringLiteral || indexedAccess.argumentExpression.kind === SyntaxKind.NumericLiteral) { const literal = indexedAccess.argumentExpression; literal.text = internIdentifier(literal.text); } } parseExpected(SyntaxKind.CloseBracketToken); expression = finishNode(indexedAccess); continue; } if (token() === SyntaxKind.NoSubstitutionTemplateLiteral || token() === SyntaxKind.TemplateHead) { const tagExpression = createNode(SyntaxKind.TaggedTemplateExpression, expression.pos); tagExpression.tag = expression; tagExpression.template = token() === SyntaxKind.NoSubstitutionTemplateLiteral ? parseLiteralNode() : parseTemplateExpression(); expression = finishNode(tagExpression); continue; } return expression; } } function parseCallExpressionRest(expression: LeftHandSideExpression): LeftHandSideExpression { while (true) { expression = parseMemberExpressionRest(expression); if (token() === SyntaxKind.LessThanToken) { // See if this is the start of a generic invocation. If so, consume it and // keep checking for postfix expressions. Otherwise, it's just a '<' that's // part of an arithmetic expression. Break out so we consume it higher in the // stack. const typeArguments = tryParse(parseTypeArgumentsInExpression); if (!typeArguments) { return expression; } const callExpr = createNode(SyntaxKind.CallExpression, expression.pos); callExpr.expression = expression; callExpr.typeArguments = typeArguments; callExpr.arguments = parseArgumentList(); expression = finishNode(callExpr); continue; } else if (token() === SyntaxKind.OpenParenToken) { const callExpr = createNode(SyntaxKind.CallExpression, expression.pos); callExpr.expression = expression; callExpr.arguments = parseArgumentList(); expression = finishNode(callExpr); continue; } return expression; } } function parseArgumentList() { parseExpected(SyntaxKind.OpenParenToken); const result = parseDelimitedList(ParsingContext.ArgumentExpressions, parseArgumentExpression); parseExpected(SyntaxKind.CloseParenToken); return result; } function parseTypeArgumentsInExpression() { if (!parseOptional(SyntaxKind.LessThanToken)) { return undefined; } const typeArguments = parseDelimitedList(ParsingContext.TypeArguments, parseType); if (!parseExpected(SyntaxKind.GreaterThanToken)) { // If it doesn't have the closing `>` then it's definitely not an type argument list. return undefined; } // If we have a '<', then only parse this as a argument list if the type arguments // are complete and we have an open paren. if we don't, rewind and return nothing. return typeArguments && canFollowTypeArgumentsInExpression() ? typeArguments : undefined; } function canFollowTypeArgumentsInExpression(): boolean { switch (token()) { case SyntaxKind.OpenParenToken: // foo( // this case are the only case where this token can legally follow a type argument // list. So we definitely want to treat this as a type arg list. case SyntaxKind.DotToken: // foo. case SyntaxKind.CloseParenToken: // foo) case SyntaxKind.CloseBracketToken: // foo] case SyntaxKind.ColonToken: // foo: case SyntaxKind.SemicolonToken: // foo; case SyntaxKind.QuestionToken: // foo? case SyntaxKind.EqualsEqualsToken: // foo == case SyntaxKind.EqualsEqualsEqualsToken: // foo === case SyntaxKind.ExclamationEqualsToken: // foo != case SyntaxKind.ExclamationEqualsEqualsToken: // foo !== case SyntaxKind.AmpersandAmpersandToken: // foo && case SyntaxKind.BarBarToken: // foo || case SyntaxKind.CaretToken: // foo ^ case SyntaxKind.AmpersandToken: // foo & case SyntaxKind.BarToken: // foo | case SyntaxKind.CloseBraceToken: // foo } case SyntaxKind.EndOfFileToken: // foo // these cases can't legally follow a type arg list. However, they're not legal // expressions either. The user is probably in the middle of a generic type. So // treat it as such. return true; case SyntaxKind.CommaToken: // foo, case SyntaxKind.OpenBraceToken: // foo { // We don't want to treat these as type arguments. Otherwise we'll parse this // as an invocation expression. Instead, we want to parse out the expression // in isolation from the type arguments. default: // Anything else treat as an expression. return false; } } function parsePrimaryExpression(): PrimaryExpression { switch (token()) { case SyntaxKind.NumericLiteral: case SyntaxKind.StringLiteral: case SyntaxKind.NoSubstitutionTemplateLiteral: return parseLiteralNode(); case SyntaxKind.ThisKeyword: case SyntaxKind.SuperKeyword: case SyntaxKind.NullKeyword: case SyntaxKind.TrueKeyword: case SyntaxKind.FalseKeyword: return parseTokenNode(); case SyntaxKind.OpenParenToken: return parseParenthesizedExpression(); case SyntaxKind.OpenBracketToken: return parseArrayLiteralExpression(); case SyntaxKind.OpenBraceToken: return parseObjectLiteralExpression(); case SyntaxKind.AsyncKeyword: // Async arrow functions are parsed earlier in parseAssignmentExpressionOrHigher. // If we encounter `async [no LineTerminator here] function` then this is an async // function; otherwise, its an identifier. if (!lookAhead(nextTokenIsFunctionKeywordOnSameLine)) { break; } return parseFunctionExpression(); case SyntaxKind.ClassKeyword: return parseClassExpression(); case SyntaxKind.FunctionKeyword: return parseFunctionExpression(); case SyntaxKind.NewKeyword: return parseNewExpression(); case SyntaxKind.SlashToken: case SyntaxKind.SlashEqualsToken: if (reScanSlashToken() === SyntaxKind.RegularExpressionLiteral) { return parseLiteralNode(); } break; case SyntaxKind.TemplateHead: return parseTemplateExpression(); } return parseIdentifier(Diagnostics.Expression_expected); } function parseParenthesizedExpression(): ParenthesizedExpression { const node = createNodeWithJSDoc(SyntaxKind.ParenthesizedExpression); parseExpected(SyntaxKind.OpenParenToken); node.expression = allowInAnd(parseExpression); parseExpected(SyntaxKind.CloseParenToken); return finishNode(node); } function parseSpreadElement(): Expression { const node = createNode(SyntaxKind.SpreadElement); parseExpected(SyntaxKind.DotDotDotToken); node.expression = parseAssignmentExpressionOrHigher(); return finishNode(node); } function parseArgumentOrArrayLiteralElement(): Expression { return token() === SyntaxKind.DotDotDotToken ? parseSpreadElement() : token() === SyntaxKind.CommaToken ? createNode(SyntaxKind.OmittedExpression) : parseAssignmentExpressionOrHigher(); } function parseArgumentExpression(): Expression { return doOutsideOfContext(disallowInAndDecoratorContext, parseArgumentOrArrayLiteralElement); } function parseArrayLiteralExpression(): ArrayLiteralExpression { const node = createNode(SyntaxKind.ArrayLiteralExpression); parseExpected(SyntaxKind.OpenBracketToken); if (scanner.hasPrecedingLineBreak()) { node.multiLine = true; } node.elements = parseDelimitedList(ParsingContext.ArrayLiteralMembers, parseArgumentOrArrayLiteralElement); parseExpected(SyntaxKind.CloseBracketToken); return finishNode(node); } function parseObjectLiteralElement(): ObjectLiteralElementLike { const node = createNodeWithJSDoc(SyntaxKind.Unknown); if (parseOptionalToken(SyntaxKind.DotDotDotToken)) { node.kind = SyntaxKind.SpreadAssignment; (node).expression = parseAssignmentExpressionOrHigher(); return finishNode(node); } node.decorators = parseDecorators(); node.modifiers = parseModifiers(); if (parseContextualModifier(SyntaxKind.GetKeyword)) { return parseAccessorDeclaration(node, SyntaxKind.GetAccessor); } if (parseContextualModifier(SyntaxKind.SetKeyword)) { return parseAccessorDeclaration(node, SyntaxKind.SetAccessor); } const asteriskToken = parseOptionalToken(SyntaxKind.AsteriskToken); const tokenIsIdentifier = isIdentifier(); node.name = parsePropertyName(); // Disallowing of optional property assignments happens in the grammar checker. (node).questionToken = parseOptionalToken(SyntaxKind.QuestionToken); if (asteriskToken || token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.LessThanToken) { return parseMethodDeclaration(node, asteriskToken); } // check if it is short-hand property assignment or normal property assignment // NOTE: if token is EqualsToken it is interpreted as CoverInitializedName production // CoverInitializedName[Yield] : // IdentifierReference[?Yield] Initializer[In, ?Yield] // this is necessary because ObjectLiteral productions are also used to cover grammar for ObjectAssignmentPattern const isShorthandPropertyAssignment = tokenIsIdentifier && (token() === SyntaxKind.CommaToken || token() === SyntaxKind.CloseBraceToken || token() === SyntaxKind.EqualsToken); if (isShorthandPropertyAssignment) { node.kind = SyntaxKind.ShorthandPropertyAssignment; const equalsToken = parseOptionalToken(SyntaxKind.EqualsToken); if (equalsToken) { (node).equalsToken = equalsToken; (node).objectAssignmentInitializer = allowInAnd(parseAssignmentExpressionOrHigher); } } else { node.kind = SyntaxKind.PropertyAssignment; parseExpected(SyntaxKind.ColonToken); (node).initializer = allowInAnd(parseAssignmentExpressionOrHigher); } return finishNode(node); } function parseObjectLiteralExpression(): ObjectLiteralExpression { const node = createNode(SyntaxKind.ObjectLiteralExpression); parseExpected(SyntaxKind.OpenBraceToken); if (scanner.hasPrecedingLineBreak()) { node.multiLine = true; } node.properties = parseDelimitedList(ParsingContext.ObjectLiteralMembers, parseObjectLiteralElement, /*considerSemicolonAsDelimiter*/ true); parseExpected(SyntaxKind.CloseBraceToken); return finishNode(node); } function parseFunctionExpression(): FunctionExpression { // GeneratorExpression: // function* BindingIdentifier [Yield][opt](FormalParameters[Yield]){ GeneratorBody } // // FunctionExpression: // function BindingIdentifier[opt](FormalParameters){ FunctionBody } const saveDecoratorContext = inDecoratorContext(); if (saveDecoratorContext) { setDecoratorContext(/*val*/ false); } const node = createNodeWithJSDoc(SyntaxKind.FunctionExpression); node.modifiers = parseModifiers(); parseExpected(SyntaxKind.FunctionKeyword); node.asteriskToken = parseOptionalToken(SyntaxKind.AsteriskToken); const isGenerator = node.asteriskToken ? SignatureFlags.Yield : SignatureFlags.None; const isAsync = hasModifier(node, ModifierFlags.Async) ? SignatureFlags.Await : SignatureFlags.None; node.name = isGenerator && isAsync ? doInYieldAndAwaitContext(parseOptionalIdentifier) : isGenerator ? doInYieldContext(parseOptionalIdentifier) : isAsync ? doInAwaitContext(parseOptionalIdentifier) : parseOptionalIdentifier(); fillSignature(SyntaxKind.ColonToken, isGenerator | isAsync, node); node.body = parseFunctionBlock(isGenerator | isAsync); if (saveDecoratorContext) { setDecoratorContext(/*val*/ true); } return finishNode(node); } function parseOptionalIdentifier(): Identifier | undefined { return isIdentifier() ? parseIdentifier() : undefined; } function parseNewExpression(): NewExpression | MetaProperty { const fullStart = scanner.getStartPos(); parseExpected(SyntaxKind.NewKeyword); if (parseOptional(SyntaxKind.DotToken)) { const node = createNode(SyntaxKind.MetaProperty, fullStart); node.keywordToken = SyntaxKind.NewKeyword; node.name = parseIdentifierName(); return finishNode(node); } const node = createNode(SyntaxKind.NewExpression, fullStart); node.expression = parseMemberExpressionOrHigher(); node.typeArguments = tryParse(parseTypeArgumentsInExpression); if (node.typeArguments || token() === SyntaxKind.OpenParenToken) { node.arguments = parseArgumentList(); } return finishNode(node); } // STATEMENTS function parseBlock(ignoreMissingOpenBrace: boolean, diagnosticMessage?: DiagnosticMessage): Block { const node = createNode(SyntaxKind.Block); if (parseExpected(SyntaxKind.OpenBraceToken, diagnosticMessage) || ignoreMissingOpenBrace) { if (scanner.hasPrecedingLineBreak()) { node.multiLine = true; } node.statements = parseList(ParsingContext.BlockStatements, parseStatement); parseExpected(SyntaxKind.CloseBraceToken); } else { node.statements = createMissingList(); } return finishNode(node); } function parseFunctionBlock(flags: SignatureFlags, diagnosticMessage?: DiagnosticMessage): Block { const savedYieldContext = inYieldContext(); setYieldContext(!!(flags & SignatureFlags.Yield)); const savedAwaitContext = inAwaitContext(); setAwaitContext(!!(flags & SignatureFlags.Await)); // We may be in a [Decorator] context when parsing a function expression or // arrow function. The body of the function is not in [Decorator] context. const saveDecoratorContext = inDecoratorContext(); if (saveDecoratorContext) { setDecoratorContext(/*val*/ false); } const block = parseBlock(!!(flags & SignatureFlags.IgnoreMissingOpenBrace), diagnosticMessage); if (saveDecoratorContext) { setDecoratorContext(/*val*/ true); } setYieldContext(savedYieldContext); setAwaitContext(savedAwaitContext); return block; } function parseEmptyStatement(): Statement { const node = createNode(SyntaxKind.EmptyStatement); parseExpected(SyntaxKind.SemicolonToken); return finishNode(node); } function parseIfStatement(): IfStatement { const node = createNode(SyntaxKind.IfStatement); parseExpected(SyntaxKind.IfKeyword); parseExpected(SyntaxKind.OpenParenToken); node.expression = allowInAnd(parseExpression); parseExpected(SyntaxKind.CloseParenToken); node.thenStatement = parseStatement(); node.elseStatement = parseOptional(SyntaxKind.ElseKeyword) ? parseStatement() : undefined; return finishNode(node); } function parseDoStatement(): DoStatement { const node = createNode(SyntaxKind.DoStatement); parseExpected(SyntaxKind.DoKeyword); node.statement = parseStatement(); parseExpected(SyntaxKind.WhileKeyword); parseExpected(SyntaxKind.OpenParenToken); node.expression = allowInAnd(parseExpression); parseExpected(SyntaxKind.CloseParenToken); // From: https://mail.mozilla.org/pipermail/es-discuss/2011-August/016188.html // 157 min --- All allen at wirfs-brock.com CONF --- "do{;}while(false)false" prohibited in // spec but allowed in consensus reality. Approved -- this is the de-facto standard whereby // do;while(0)x will have a semicolon inserted before x. parseOptional(SyntaxKind.SemicolonToken); return finishNode(node); } function parseWhileStatement(): WhileStatement { const node = createNode(SyntaxKind.WhileStatement); parseExpected(SyntaxKind.WhileKeyword); parseExpected(SyntaxKind.OpenParenToken); node.expression = allowInAnd(parseExpression); parseExpected(SyntaxKind.CloseParenToken); node.statement = parseStatement(); return finishNode(node); } function parseForOrForInOrForOfStatement(): Statement { const pos = getNodePos(); parseExpected(SyntaxKind.ForKeyword); const awaitToken = parseOptionalToken(SyntaxKind.AwaitKeyword); parseExpected(SyntaxKind.OpenParenToken); let initializer: VariableDeclarationList | Expression = undefined; if (token() !== SyntaxKind.SemicolonToken) { if (token() === SyntaxKind.VarKeyword || token() === SyntaxKind.LetKeyword || token() === SyntaxKind.ConstKeyword) { initializer = parseVariableDeclarationList(/*inForStatementInitializer*/ true); } else { initializer = disallowInAnd(parseExpression); } } let forOrForInOrForOfStatement: IterationStatement; if (awaitToken ? parseExpected(SyntaxKind.OfKeyword) : parseOptional(SyntaxKind.OfKeyword)) { const forOfStatement = createNode(SyntaxKind.ForOfStatement, pos); forOfStatement.awaitModifier = awaitToken; forOfStatement.initializer = initializer; forOfStatement.expression = allowInAnd(parseAssignmentExpressionOrHigher); parseExpected(SyntaxKind.CloseParenToken); forOrForInOrForOfStatement = forOfStatement; } else if (parseOptional(SyntaxKind.InKeyword)) { const forInStatement = createNode(SyntaxKind.ForInStatement, pos); forInStatement.initializer = initializer; forInStatement.expression = allowInAnd(parseExpression); parseExpected(SyntaxKind.CloseParenToken); forOrForInOrForOfStatement = forInStatement; } else { const forStatement = createNode(SyntaxKind.ForStatement, pos); forStatement.initializer = initializer; parseExpected(SyntaxKind.SemicolonToken); if (token() !== SyntaxKind.SemicolonToken && token() !== SyntaxKind.CloseParenToken) { forStatement.condition = allowInAnd(parseExpression); } parseExpected(SyntaxKind.SemicolonToken); if (token() !== SyntaxKind.CloseParenToken) { forStatement.incrementor = allowInAnd(parseExpression); } parseExpected(SyntaxKind.CloseParenToken); forOrForInOrForOfStatement = forStatement; } forOrForInOrForOfStatement.statement = parseStatement(); return finishNode(forOrForInOrForOfStatement); } function parseBreakOrContinueStatement(kind: SyntaxKind): BreakOrContinueStatement { const node = createNode(kind); parseExpected(kind === SyntaxKind.BreakStatement ? SyntaxKind.BreakKeyword : SyntaxKind.ContinueKeyword); if (!canParseSemicolon()) { node.label = parseIdentifier(); } parseSemicolon(); return finishNode(node); } function parseReturnStatement(): ReturnStatement { const node = createNode(SyntaxKind.ReturnStatement); parseExpected(SyntaxKind.ReturnKeyword); if (!canParseSemicolon()) { node.expression = allowInAnd(parseExpression); } parseSemicolon(); return finishNode(node); } function parseWithStatement(): WithStatement { const node = createNode(SyntaxKind.WithStatement); parseExpected(SyntaxKind.WithKeyword); parseExpected(SyntaxKind.OpenParenToken); node.expression = allowInAnd(parseExpression); parseExpected(SyntaxKind.CloseParenToken); node.statement = doInsideOfContext(NodeFlags.InWithStatement, parseStatement); return finishNode(node); } function parseCaseClause(): CaseClause { const node = createNode(SyntaxKind.CaseClause); parseExpected(SyntaxKind.CaseKeyword); node.expression = allowInAnd(parseExpression); parseExpected(SyntaxKind.ColonToken); node.statements = parseList(ParsingContext.SwitchClauseStatements, parseStatement); return finishNode(node); } function parseDefaultClause(): DefaultClause { const node = createNode(SyntaxKind.DefaultClause); parseExpected(SyntaxKind.DefaultKeyword); parseExpected(SyntaxKind.ColonToken); node.statements = parseList(ParsingContext.SwitchClauseStatements, parseStatement); return finishNode(node); } function parseCaseOrDefaultClause(): CaseOrDefaultClause { return token() === SyntaxKind.CaseKeyword ? parseCaseClause() : parseDefaultClause(); } function parseSwitchStatement(): SwitchStatement { const node = createNode(SyntaxKind.SwitchStatement); parseExpected(SyntaxKind.SwitchKeyword); parseExpected(SyntaxKind.OpenParenToken); node.expression = allowInAnd(parseExpression); parseExpected(SyntaxKind.CloseParenToken); const caseBlock = createNode(SyntaxKind.CaseBlock); parseExpected(SyntaxKind.OpenBraceToken); caseBlock.clauses = parseList(ParsingContext.SwitchClauses, parseCaseOrDefaultClause); parseExpected(SyntaxKind.CloseBraceToken); node.caseBlock = finishNode(caseBlock); return finishNode(node); } function parseThrowStatement(): ThrowStatement { // ThrowStatement[Yield] : // throw [no LineTerminator here]Expression[In, ?Yield]; // Because of automatic semicolon insertion, we need to report error if this // throw could be terminated with a semicolon. Note: we can't call 'parseExpression' // directly as that might consume an expression on the following line. // We just return 'undefined' in that case. The actual error will be reported in the // grammar walker. const node = createNode(SyntaxKind.ThrowStatement); parseExpected(SyntaxKind.ThrowKeyword); node.expression = scanner.hasPrecedingLineBreak() ? undefined : allowInAnd(parseExpression); parseSemicolon(); return finishNode(node); } // TODO: Review for error recovery function parseTryStatement(): TryStatement { const node = createNode(SyntaxKind.TryStatement); parseExpected(SyntaxKind.TryKeyword); node.tryBlock = parseBlock(/*ignoreMissingOpenBrace*/ false); node.catchClause = token() === SyntaxKind.CatchKeyword ? parseCatchClause() : undefined; // If we don't have a catch clause, then we must have a finally clause. Try to parse // one out no matter what. if (!node.catchClause || token() === SyntaxKind.FinallyKeyword) { parseExpected(SyntaxKind.FinallyKeyword); node.finallyBlock = parseBlock(/*ignoreMissingOpenBrace*/ false); } return finishNode(node); } function parseCatchClause(): CatchClause { const result = createNode(SyntaxKind.CatchClause); parseExpected(SyntaxKind.CatchKeyword); if (parseOptional(SyntaxKind.OpenParenToken)) { result.variableDeclaration = parseVariableDeclaration(); parseExpected(SyntaxKind.CloseParenToken); } else { // Keep shape of node to avoid degrading performance. result.variableDeclaration = undefined; } result.block = parseBlock(/*ignoreMissingOpenBrace*/ false); return finishNode(result); } function parseDebuggerStatement(): Statement { const node = createNode(SyntaxKind.DebuggerStatement); parseExpected(SyntaxKind.DebuggerKeyword); parseSemicolon(); return finishNode(node); } function parseExpressionOrLabeledStatement(): ExpressionStatement | LabeledStatement { // Avoiding having to do the lookahead for a labeled statement by just trying to parse // out an expression, seeing if it is identifier and then seeing if it is followed by // a colon. const node = createNodeWithJSDoc(SyntaxKind.Unknown); const expression = allowInAnd(parseExpression); if (expression.kind === SyntaxKind.Identifier && parseOptional(SyntaxKind.ColonToken)) { node.kind = SyntaxKind.LabeledStatement; (node).label = expression; (node).statement = parseStatement(); } else { node.kind = SyntaxKind.ExpressionStatement; (node).expression = expression; parseSemicolon(); } return finishNode(node); } function nextTokenIsIdentifierOrKeywordOnSameLine() { nextToken(); return tokenIsIdentifierOrKeyword(token()) && !scanner.hasPrecedingLineBreak(); } function nextTokenIsClassKeywordOnSameLine() { nextToken(); return token() === SyntaxKind.ClassKeyword && !scanner.hasPrecedingLineBreak(); } function nextTokenIsFunctionKeywordOnSameLine() { nextToken(); return token() === SyntaxKind.FunctionKeyword && !scanner.hasPrecedingLineBreak(); } function nextTokenIsIdentifierOrKeywordOrLiteralOnSameLine() { nextToken(); return (tokenIsIdentifierOrKeyword(token()) || token() === SyntaxKind.NumericLiteral || token() === SyntaxKind.StringLiteral) && !scanner.hasPrecedingLineBreak(); } function isDeclaration(): boolean { while (true) { switch (token()) { case SyntaxKind.VarKeyword: case SyntaxKind.LetKeyword: case SyntaxKind.ConstKeyword: case SyntaxKind.FunctionKeyword: case SyntaxKind.ClassKeyword: case SyntaxKind.EnumKeyword: return true; // 'declare', 'module', 'namespace', 'interface'* and 'type' are all legal JavaScript identifiers; // however, an identifier cannot be followed by another identifier on the same line. This is what we // count on to parse out the respective declarations. For instance, we exploit this to say that // // namespace n // // can be none other than the beginning of a namespace declaration, but need to respect that JavaScript sees // // namespace // n // // as the identifier 'namespace' on one line followed by the identifier 'n' on another. // We need to look one token ahead to see if it permissible to try parsing a declaration. // // *Note*: 'interface' is actually a strict mode reserved word. So while // // "use strict" // interface // I {} // // could be legal, it would add complexity for very little gain. case SyntaxKind.InterfaceKeyword: case SyntaxKind.TypeKeyword: return nextTokenIsIdentifierOnSameLine(); case SyntaxKind.ModuleKeyword: case SyntaxKind.NamespaceKeyword: return nextTokenIsIdentifierOrStringLiteralOnSameLine(); case SyntaxKind.AbstractKeyword: case SyntaxKind.AsyncKeyword: case SyntaxKind.DeclareKeyword: case SyntaxKind.PrivateKeyword: case SyntaxKind.ProtectedKeyword: case SyntaxKind.PublicKeyword: case SyntaxKind.ReadonlyKeyword: nextToken(); // ASI takes effect for this modifier. if (scanner.hasPrecedingLineBreak()) { return false; } continue; case SyntaxKind.GlobalKeyword: nextToken(); return token() === SyntaxKind.OpenBraceToken || token() === SyntaxKind.Identifier || token() === SyntaxKind.ExportKeyword; case SyntaxKind.ImportKeyword: nextToken(); return token() === SyntaxKind.StringLiteral || token() === SyntaxKind.AsteriskToken || token() === SyntaxKind.OpenBraceToken || tokenIsIdentifierOrKeyword(token()); case SyntaxKind.ExportKeyword: nextToken(); if (token() === SyntaxKind.EqualsToken || token() === SyntaxKind.AsteriskToken || token() === SyntaxKind.OpenBraceToken || token() === SyntaxKind.DefaultKeyword || token() === SyntaxKind.AsKeyword) { return true; } continue; case SyntaxKind.StaticKeyword: nextToken(); continue; default: return false; } } } function isStartOfDeclaration(): boolean { return lookAhead(isDeclaration); } function isStartOfStatement(): boolean { switch (token()) { case SyntaxKind.AtToken: case SyntaxKind.SemicolonToken: case SyntaxKind.OpenBraceToken: case SyntaxKind.VarKeyword: case SyntaxKind.LetKeyword: case SyntaxKind.FunctionKeyword: case SyntaxKind.ClassKeyword: case SyntaxKind.EnumKeyword: case SyntaxKind.IfKeyword: case SyntaxKind.DoKeyword: case SyntaxKind.WhileKeyword: case SyntaxKind.ForKeyword: case SyntaxKind.ContinueKeyword: case SyntaxKind.BreakKeyword: case SyntaxKind.ReturnKeyword: case SyntaxKind.WithKeyword: case SyntaxKind.SwitchKeyword: case SyntaxKind.ThrowKeyword: case SyntaxKind.TryKeyword: case SyntaxKind.DebuggerKeyword: // 'catch' and 'finally' do not actually indicate that the code is part of a statement, // however, we say they are here so that we may gracefully parse them and error later. case SyntaxKind.CatchKeyword: case SyntaxKind.FinallyKeyword: return true; case SyntaxKind.ImportKeyword: return isStartOfDeclaration() || lookAhead(nextTokenIsOpenParenOrLessThan); case SyntaxKind.ConstKeyword: case SyntaxKind.ExportKeyword: return isStartOfDeclaration(); case SyntaxKind.AsyncKeyword: case SyntaxKind.DeclareKeyword: case SyntaxKind.InterfaceKeyword: case SyntaxKind.ModuleKeyword: case SyntaxKind.NamespaceKeyword: case SyntaxKind.TypeKeyword: case SyntaxKind.GlobalKeyword: // When these don't start a declaration, they're an identifier in an expression statement return true; case SyntaxKind.PublicKeyword: case SyntaxKind.PrivateKeyword: case SyntaxKind.ProtectedKeyword: case SyntaxKind.StaticKeyword: case SyntaxKind.ReadonlyKeyword: // When these don't start a declaration, they may be the start of a class member if an identifier // immediately follows. Otherwise they're an identifier in an expression statement. return isStartOfDeclaration() || !lookAhead(nextTokenIsIdentifierOrKeywordOnSameLine); default: return isStartOfExpression(); } } function nextTokenIsIdentifierOrStartOfDestructuring() { nextToken(); return isIdentifier() || token() === SyntaxKind.OpenBraceToken || token() === SyntaxKind.OpenBracketToken; } function isLetDeclaration() { // In ES6 'let' always starts a lexical declaration if followed by an identifier or { // or [. return lookAhead(nextTokenIsIdentifierOrStartOfDestructuring); } function parseStatement(): Statement { switch (token()) { case SyntaxKind.SemicolonToken: return parseEmptyStatement(); case SyntaxKind.OpenBraceToken: return parseBlock(/*ignoreMissingOpenBrace*/ false); case SyntaxKind.VarKeyword: return parseVariableStatement(createNodeWithJSDoc(SyntaxKind.VariableDeclaration)); case SyntaxKind.LetKeyword: if (isLetDeclaration()) { return parseVariableStatement(createNodeWithJSDoc(SyntaxKind.VariableDeclaration)); } break; case SyntaxKind.FunctionKeyword: return parseFunctionDeclaration(createNodeWithJSDoc(SyntaxKind.FunctionDeclaration)); case SyntaxKind.ClassKeyword: return parseClassDeclaration(createNodeWithJSDoc(SyntaxKind.ClassDeclaration)); case SyntaxKind.IfKeyword: return parseIfStatement(); case SyntaxKind.DoKeyword: return parseDoStatement(); case SyntaxKind.WhileKeyword: return parseWhileStatement(); case SyntaxKind.ForKeyword: return parseForOrForInOrForOfStatement(); case SyntaxKind.ContinueKeyword: return parseBreakOrContinueStatement(SyntaxKind.ContinueStatement); case SyntaxKind.BreakKeyword: return parseBreakOrContinueStatement(SyntaxKind.BreakStatement); case SyntaxKind.ReturnKeyword: return parseReturnStatement(); case SyntaxKind.WithKeyword: return parseWithStatement(); case SyntaxKind.SwitchKeyword: return parseSwitchStatement(); case SyntaxKind.ThrowKeyword: return parseThrowStatement(); case SyntaxKind.TryKeyword: // Include 'catch' and 'finally' for error recovery. case SyntaxKind.CatchKeyword: case SyntaxKind.FinallyKeyword: return parseTryStatement(); case SyntaxKind.DebuggerKeyword: return parseDebuggerStatement(); case SyntaxKind.AtToken: return parseDeclaration(); case SyntaxKind.AsyncKeyword: case SyntaxKind.InterfaceKeyword: case SyntaxKind.TypeKeyword: case SyntaxKind.ModuleKeyword: case SyntaxKind.NamespaceKeyword: case SyntaxKind.DeclareKeyword: case SyntaxKind.ConstKeyword: case SyntaxKind.EnumKeyword: case SyntaxKind.ExportKeyword: case SyntaxKind.ImportKeyword: case SyntaxKind.PrivateKeyword: case SyntaxKind.ProtectedKeyword: case SyntaxKind.PublicKeyword: case SyntaxKind.AbstractKeyword: case SyntaxKind.StaticKeyword: case SyntaxKind.ReadonlyKeyword: case SyntaxKind.GlobalKeyword: if (isStartOfDeclaration()) { return parseDeclaration(); } break; } return parseExpressionOrLabeledStatement(); } function isDeclareModifier(modifier: Modifier) { return modifier.kind === SyntaxKind.DeclareKeyword; } function parseDeclaration(): Statement { const node = createNodeWithJSDoc(SyntaxKind.Unknown); node.decorators = parseDecorators(); node.modifiers = parseModifiers(); if (some(node.modifiers, isDeclareModifier)) { for (const m of node.modifiers) { m.flags |= NodeFlags.Ambient; } return doInsideOfContext(NodeFlags.Ambient, () => parseDeclarationWorker(node)); } else { return parseDeclarationWorker(node); } } function parseDeclarationWorker(node: Statement): Statement { switch (token()) { case SyntaxKind.VarKeyword: case SyntaxKind.LetKeyword: case SyntaxKind.ConstKeyword: return parseVariableStatement(node); case SyntaxKind.FunctionKeyword: return parseFunctionDeclaration(node); case SyntaxKind.ClassKeyword: return parseClassDeclaration(node); case SyntaxKind.InterfaceKeyword: return parseInterfaceDeclaration(node); case SyntaxKind.TypeKeyword: return parseTypeAliasDeclaration(node); case SyntaxKind.EnumKeyword: return parseEnumDeclaration(node); case SyntaxKind.GlobalKeyword: case SyntaxKind.ModuleKeyword: case SyntaxKind.NamespaceKeyword: return parseModuleDeclaration(node); case SyntaxKind.ImportKeyword: return parseImportDeclarationOrImportEqualsDeclaration(node); case SyntaxKind.ExportKeyword: nextToken(); switch (token()) { case SyntaxKind.DefaultKeyword: case SyntaxKind.EqualsToken: return parseExportAssignment(node); case SyntaxKind.AsKeyword: return parseNamespaceExportDeclaration(node); default: return parseExportDeclaration(node); } default: if (node.decorators || node.modifiers) { // We reached this point because we encountered decorators and/or modifiers and assumed a declaration // would follow. For recovery and error reporting purposes, return an incomplete declaration. const missing = createMissingNode(SyntaxKind.MissingDeclaration, /*reportAtCurrentPosition*/ true, Diagnostics.Declaration_expected); missing.pos = node.pos; missing.decorators = node.decorators; missing.modifiers = node.modifiers; return finishNode(missing); } } } function nextTokenIsIdentifierOrStringLiteralOnSameLine() { nextToken(); return !scanner.hasPrecedingLineBreak() && (isIdentifier() || token() === SyntaxKind.StringLiteral); } function parseFunctionBlockOrSemicolon(flags: SignatureFlags, diagnosticMessage?: DiagnosticMessage): Block { if (token() !== SyntaxKind.OpenBraceToken && canParseSemicolon()) { parseSemicolon(); return; } return parseFunctionBlock(flags, diagnosticMessage); } // DECLARATIONS function parseArrayBindingElement(): ArrayBindingElement { if (token() === SyntaxKind.CommaToken) { return createNode(SyntaxKind.OmittedExpression); } const node = createNode(SyntaxKind.BindingElement); node.dotDotDotToken = parseOptionalToken(SyntaxKind.DotDotDotToken); node.name = parseIdentifierOrPattern(); node.initializer = parseInitializer(); return finishNode(node); } function parseObjectBindingElement(): BindingElement { const node = createNode(SyntaxKind.BindingElement); node.dotDotDotToken = parseOptionalToken(SyntaxKind.DotDotDotToken); const tokenIsIdentifier = isIdentifier(); const propertyName = parsePropertyName(); if (tokenIsIdentifier && token() !== SyntaxKind.ColonToken) { node.name = propertyName; } else { parseExpected(SyntaxKind.ColonToken); node.propertyName = propertyName; node.name = parseIdentifierOrPattern(); } node.initializer = parseInitializer(); return finishNode(node); } function parseObjectBindingPattern(): ObjectBindingPattern { const node = createNode(SyntaxKind.ObjectBindingPattern); parseExpected(SyntaxKind.OpenBraceToken); node.elements = parseDelimitedList(ParsingContext.ObjectBindingElements, parseObjectBindingElement); parseExpected(SyntaxKind.CloseBraceToken); return finishNode(node); } function parseArrayBindingPattern(): ArrayBindingPattern { const node = createNode(SyntaxKind.ArrayBindingPattern); parseExpected(SyntaxKind.OpenBracketToken); node.elements = parseDelimitedList(ParsingContext.ArrayBindingElements, parseArrayBindingElement); parseExpected(SyntaxKind.CloseBracketToken); return finishNode(node); } function isIdentifierOrPattern() { return token() === SyntaxKind.OpenBraceToken || token() === SyntaxKind.OpenBracketToken || isIdentifier(); } function parseIdentifierOrPattern(): Identifier | BindingPattern { if (token() === SyntaxKind.OpenBracketToken) { return parseArrayBindingPattern(); } if (token() === SyntaxKind.OpenBraceToken) { return parseObjectBindingPattern(); } return parseIdentifier(); } function parseVariableDeclarationAllowExclamation() { return parseVariableDeclaration(/*allowExclamation*/ true); } function parseVariableDeclaration(allowExclamation?: boolean): VariableDeclaration { const node = createNode(SyntaxKind.VariableDeclaration); node.name = parseIdentifierOrPattern(); if (allowExclamation && node.name.kind === SyntaxKind.Identifier && token() === SyntaxKind.ExclamationToken && !scanner.hasPrecedingLineBreak()) { node.exclamationToken = parseTokenNode(); } node.type = parseTypeAnnotation(); if (!isInOrOfKeyword(token())) { node.initializer = parseInitializer(); } return finishNode(node); } function parseVariableDeclarationList(inForStatementInitializer: boolean): VariableDeclarationList { const node = createNode(SyntaxKind.VariableDeclarationList); switch (token()) { case SyntaxKind.VarKeyword: break; case SyntaxKind.LetKeyword: node.flags |= NodeFlags.Let; break; case SyntaxKind.ConstKeyword: node.flags |= NodeFlags.Const; break; default: Debug.fail(); } nextToken(); // The user may have written the following: // // for (let of X) { } // // In this case, we want to parse an empty declaration list, and then parse 'of' // as a keyword. The reason this is not automatic is that 'of' is a valid identifier. // So we need to look ahead to determine if 'of' should be treated as a keyword in // this context. // The checker will then give an error that there is an empty declaration list. if (token() === SyntaxKind.OfKeyword && lookAhead(canFollowContextualOfKeyword)) { node.declarations = createMissingList(); } else { const savedDisallowIn = inDisallowInContext(); setDisallowInContext(inForStatementInitializer); node.declarations = parseDelimitedList(ParsingContext.VariableDeclarations, inForStatementInitializer ? parseVariableDeclaration : parseVariableDeclarationAllowExclamation); setDisallowInContext(savedDisallowIn); } return finishNode(node); } function canFollowContextualOfKeyword(): boolean { return nextTokenIsIdentifier() && nextToken() === SyntaxKind.CloseParenToken; } function parseVariableStatement(node: VariableStatement): VariableStatement { node.kind = SyntaxKind.VariableStatement; node.declarationList = parseVariableDeclarationList(/*inForStatementInitializer*/ false); parseSemicolon(); return finishNode(node); } function parseFunctionDeclaration(node: FunctionDeclaration): FunctionDeclaration { node.kind = SyntaxKind.FunctionDeclaration; parseExpected(SyntaxKind.FunctionKeyword); node.asteriskToken = parseOptionalToken(SyntaxKind.AsteriskToken); node.name = hasModifier(node, ModifierFlags.Default) ? parseOptionalIdentifier() : parseIdentifier(); const isGenerator = node.asteriskToken ? SignatureFlags.Yield : SignatureFlags.None; const isAsync = hasModifier(node, ModifierFlags.Async) ? SignatureFlags.Await : SignatureFlags.None; fillSignature(SyntaxKind.ColonToken, isGenerator | isAsync, node); node.body = parseFunctionBlockOrSemicolon(isGenerator | isAsync, Diagnostics.or_expected); return finishNode(node); } function parseConstructorDeclaration(node: ConstructorDeclaration): ConstructorDeclaration { node.kind = SyntaxKind.Constructor; parseExpected(SyntaxKind.ConstructorKeyword); fillSignature(SyntaxKind.ColonToken, SignatureFlags.None, node); node.body = parseFunctionBlockOrSemicolon(SignatureFlags.None, Diagnostics.or_expected); return finishNode(node); } function parseMethodDeclaration(node: MethodDeclaration, asteriskToken: AsteriskToken, diagnosticMessage?: DiagnosticMessage): MethodDeclaration { node.kind = SyntaxKind.MethodDeclaration; node.asteriskToken = asteriskToken; const isGenerator = asteriskToken ? SignatureFlags.Yield : SignatureFlags.None; const isAsync = hasModifier(node, ModifierFlags.Async) ? SignatureFlags.Await : SignatureFlags.None; fillSignature(SyntaxKind.ColonToken, isGenerator | isAsync, node); node.body = parseFunctionBlockOrSemicolon(isGenerator | isAsync, diagnosticMessage); return finishNode(node); } function parsePropertyDeclaration(node: PropertyDeclaration): PropertyDeclaration { node.kind = SyntaxKind.PropertyDeclaration; if (!node.questionToken && token() === SyntaxKind.ExclamationToken && !scanner.hasPrecedingLineBreak()) { node.exclamationToken = parseTokenNode(); } node.type = parseTypeAnnotation(); // For instance properties specifically, since they are evaluated inside the constructor, // we do *not * want to parse yield expressions, so we specifically turn the yield context // off. The grammar would look something like this: // // MemberVariableDeclaration[Yield]: // AccessibilityModifier_opt PropertyName TypeAnnotation_opt Initializer_opt[In]; // AccessibilityModifier_opt static_opt PropertyName TypeAnnotation_opt Initializer_opt[In, ?Yield]; // // The checker may still error in the static case to explicitly disallow the yield expression. node.initializer = hasModifier(node, ModifierFlags.Static) ? allowInAnd(parseInitializer) : doOutsideOfContext(NodeFlags.YieldContext | NodeFlags.DisallowInContext, parseInitializer); parseSemicolon(); return finishNode(node); } function parsePropertyOrMethodDeclaration(node: PropertyDeclaration | MethodDeclaration): PropertyDeclaration | MethodDeclaration { const asteriskToken = parseOptionalToken(SyntaxKind.AsteriskToken); node.name = parsePropertyName(); // Note: this is not legal as per the grammar. But we allow it in the parser and // report an error in the grammar checker. node.questionToken = parseOptionalToken(SyntaxKind.QuestionToken); if (asteriskToken || token() === SyntaxKind.OpenParenToken || token() === SyntaxKind.LessThanToken) { return parseMethodDeclaration(node, asteriskToken, Diagnostics.or_expected); } return parsePropertyDeclaration(node); } function parseAccessorDeclaration(node: AccessorDeclaration, kind: AccessorDeclaration["kind"]): AccessorDeclaration { node.kind = kind; node.name = parsePropertyName(); fillSignature(SyntaxKind.ColonToken, SignatureFlags.None, node); node.body = parseFunctionBlockOrSemicolon(SignatureFlags.None); return finishNode(node); } function isClassMemberModifier(idToken: SyntaxKind) { switch (idToken) { case SyntaxKind.PublicKeyword: case SyntaxKind.PrivateKeyword: case SyntaxKind.ProtectedKeyword: case SyntaxKind.StaticKeyword: case SyntaxKind.ReadonlyKeyword: return true; default: return false; } } function isClassMemberStart(): boolean { let idToken: SyntaxKind; if (token() === SyntaxKind.AtToken) { return true; } // Eat up all modifiers, but hold on to the last one in case it is actually an identifier. while (isModifierKind(token())) { idToken = token(); // If the idToken is a class modifier (protected, private, public, and static), it is // certain that we are starting to parse class member. This allows better error recovery // Example: // public foo() ... // true // public @dec blah ... // true; we will then report an error later // export public ... // true; we will then report an error later if (isClassMemberModifier(idToken)) { return true; } nextToken(); } if (token() === SyntaxKind.AsteriskToken) { return true; } // Try to get the first property-like token following all modifiers. // This can either be an identifier or the 'get' or 'set' keywords. if (isLiteralPropertyName()) { idToken = token(); nextToken(); } // Index signatures and computed properties are class members; we can parse. if (token() === SyntaxKind.OpenBracketToken) { return true; } // If we were able to get any potential identifier... if (idToken !== undefined) { // If we have a non-keyword identifier, or if we have an accessor, then it's safe to parse. if (!isKeyword(idToken) || idToken === SyntaxKind.SetKeyword || idToken === SyntaxKind.GetKeyword) { return true; } // If it *is* a keyword, but not an accessor, check a little farther along // to see if it should actually be parsed as a class member. switch (token()) { case SyntaxKind.OpenParenToken: // Method declaration case SyntaxKind.LessThanToken: // Generic Method declaration case SyntaxKind.ExclamationToken: // Non-null assertion on property name case SyntaxKind.ColonToken: // Type Annotation for declaration case SyntaxKind.EqualsToken: // Initializer for declaration case SyntaxKind.QuestionToken: // Not valid, but permitted so that it gets caught later on. return true; default: // Covers // - Semicolons (declaration termination) // - Closing braces (end-of-class, must be declaration) // - End-of-files (not valid, but permitted so that it gets caught later on) // - Line-breaks (enabling *automatic semicolon insertion*) return canParseSemicolon(); } } return false; } function parseDecorators(): NodeArray | undefined { let list: Decorator[] | undefined; const listPos = getNodePos(); while (true) { const decoratorStart = getNodePos(); if (!parseOptional(SyntaxKind.AtToken)) { break; } const decorator = createNode(SyntaxKind.Decorator, decoratorStart); decorator.expression = doInDecoratorContext(parseLeftHandSideExpressionOrHigher); finishNode(decorator); (list || (list = [])).push(decorator); } return list && createNodeArray(list, listPos); } /* * There are situations in which a modifier like 'const' will appear unexpectedly, such as on a class member. * In those situations, if we are entirely sure that 'const' is not valid on its own (such as when ASI takes effect * and turns it into a standalone declaration), then it is better to parse it and report an error later. * * In such situations, 'permitInvalidConstAsModifier' should be set to true. */ function parseModifiers(permitInvalidConstAsModifier?: boolean): NodeArray | undefined { let list: Modifier[]; const listPos = getNodePos(); while (true) { const modifierStart = scanner.getStartPos(); const modifierKind = token(); if (token() === SyntaxKind.ConstKeyword && permitInvalidConstAsModifier) { // We need to ensure that any subsequent modifiers appear on the same line // so that when 'const' is a standalone declaration, we don't issue an error. if (!tryParse(nextTokenIsOnSameLineAndCanFollowModifier)) { break; } } else { if (!parseAnyContextualModifier()) { break; } } const modifier = finishNode(createNode(modifierKind, modifierStart)); (list || (list = [])).push(modifier); } return list && createNodeArray(list, listPos); } function parseModifiersForArrowFunction(): NodeArray { let modifiers: NodeArray; if (token() === SyntaxKind.AsyncKeyword) { const modifierStart = scanner.getStartPos(); const modifierKind = token(); nextToken(); const modifier = finishNode(createNode(modifierKind, modifierStart)); modifiers = createNodeArray([modifier], modifierStart); } return modifiers; } function parseClassElement(): ClassElement { if (token() === SyntaxKind.SemicolonToken) { const result = createNode(SyntaxKind.SemicolonClassElement); nextToken(); return finishNode(result); } const node = createNodeWithJSDoc(SyntaxKind.Unknown); node.decorators = parseDecorators(); node.modifiers = parseModifiers(/*permitInvalidConstAsModifier*/ true); if (parseContextualModifier(SyntaxKind.GetKeyword)) { return parseAccessorDeclaration(node, SyntaxKind.GetAccessor); } if (parseContextualModifier(SyntaxKind.SetKeyword)) { return parseAccessorDeclaration(node, SyntaxKind.SetAccessor); } if (token() === SyntaxKind.ConstructorKeyword) { return parseConstructorDeclaration(node); } if (isIndexSignature()) { return parseIndexSignatureDeclaration(node); } // It is very important that we check this *after* checking indexers because // the [ token can start an index signature or a computed property name if (tokenIsIdentifierOrKeyword(token()) || token() === SyntaxKind.StringLiteral || token() === SyntaxKind.NumericLiteral || token() === SyntaxKind.AsteriskToken || token() === SyntaxKind.OpenBracketToken) { return parsePropertyOrMethodDeclaration(node); } if (node.decorators || node.modifiers) { // treat this as a property declaration with a missing name. node.name = createMissingNode(SyntaxKind.Identifier, /*reportAtCurrentPosition*/ true, Diagnostics.Declaration_expected); return parsePropertyDeclaration(node); } // 'isClassMemberStart' should have hinted not to attempt parsing. Debug.fail("Should not have attempted to parse class member declaration."); } function parseClassExpression(): ClassExpression { return parseClassDeclarationOrExpression(createNodeWithJSDoc(SyntaxKind.Unknown), SyntaxKind.ClassExpression); } function parseClassDeclaration(node: ClassLikeDeclaration): ClassDeclaration { return parseClassDeclarationOrExpression(node, SyntaxKind.ClassDeclaration); } function parseClassDeclarationOrExpression(node: ClassLikeDeclaration, kind: ClassLikeDeclaration["kind"]): ClassLikeDeclaration { node.kind = kind; parseExpected(SyntaxKind.ClassKeyword); node.name = parseNameOfClassDeclarationOrExpression(); node.typeParameters = parseTypeParameters(); node.heritageClauses = parseHeritageClauses(); if (parseExpected(SyntaxKind.OpenBraceToken)) { // ClassTail[Yield,Await] : (Modified) See 14.5 // ClassHeritage[?Yield,?Await]opt { ClassBody[?Yield,?Await]opt } node.members = parseClassMembers(); parseExpected(SyntaxKind.CloseBraceToken); } else { node.members = createMissingList(); } return finishNode(node); } function parseNameOfClassDeclarationOrExpression(): Identifier | undefined { // implements is a future reserved word so // 'class implements' might mean either // - class expression with omitted name, 'implements' starts heritage clause // - class with name 'implements' // 'isImplementsClause' helps to disambiguate between these two cases return isIdentifier() && !isImplementsClause() ? parseIdentifier() : undefined; } function isImplementsClause() { return token() === SyntaxKind.ImplementsKeyword && lookAhead(nextTokenIsIdentifierOrKeyword); } function parseHeritageClauses(): NodeArray | undefined { // ClassTail[Yield,Await] : (Modified) See 14.5 // ClassHeritage[?Yield,?Await]opt { ClassBody[?Yield,?Await]opt } if (isHeritageClause()) { return parseList(ParsingContext.HeritageClauses, parseHeritageClause); } return undefined; } function parseHeritageClause(): HeritageClause | undefined { const tok = token(); if (tok === SyntaxKind.ExtendsKeyword || tok === SyntaxKind.ImplementsKeyword) { const node = createNode(SyntaxKind.HeritageClause); node.token = tok; nextToken(); node.types = parseDelimitedList(ParsingContext.HeritageClauseElement, parseExpressionWithTypeArguments); return finishNode(node); } return undefined; } function parseExpressionWithTypeArguments(): ExpressionWithTypeArguments { const node = createNode(SyntaxKind.ExpressionWithTypeArguments); node.expression = parseLeftHandSideExpressionOrHigher(); node.typeArguments = tryParseTypeArguments(); return finishNode(node); } function tryParseTypeArguments(): NodeArray | undefined { return token() === SyntaxKind.LessThanToken ? parseBracketedList(ParsingContext.TypeArguments, parseType, SyntaxKind.LessThanToken, SyntaxKind.GreaterThanToken) : undefined; } function isHeritageClause(): boolean { return token() === SyntaxKind.ExtendsKeyword || token() === SyntaxKind.ImplementsKeyword; } function parseClassMembers(): NodeArray { return parseList(ParsingContext.ClassMembers, parseClassElement); } function parseInterfaceDeclaration(node: InterfaceDeclaration): InterfaceDeclaration { node.kind = SyntaxKind.InterfaceDeclaration; parseExpected(SyntaxKind.InterfaceKeyword); node.name = parseIdentifier(); node.typeParameters = parseTypeParameters(); node.heritageClauses = parseHeritageClauses(); node.members = parseObjectTypeMembers(); return finishNode(node); } function parseTypeAliasDeclaration(node: TypeAliasDeclaration): TypeAliasDeclaration { node.kind = SyntaxKind.TypeAliasDeclaration; parseExpected(SyntaxKind.TypeKeyword); node.name = parseIdentifier(); node.typeParameters = parseTypeParameters(); parseExpected(SyntaxKind.EqualsToken); node.type = parseType(); parseSemicolon(); return finishNode(node); } // In an ambient declaration, the grammar only allows integer literals as initializers. // In a non-ambient declaration, the grammar allows uninitialized members only in a // ConstantEnumMemberSection, which starts at the beginning of an enum declaration // or any time an integer literal initializer is encountered. function parseEnumMember(): EnumMember { const node = createNodeWithJSDoc(SyntaxKind.EnumMember); node.name = parsePropertyName(); node.initializer = allowInAnd(parseInitializer); return finishNode(node); } function parseEnumDeclaration(node: EnumDeclaration): EnumDeclaration { node.kind = SyntaxKind.EnumDeclaration; parseExpected(SyntaxKind.EnumKeyword); node.name = parseIdentifier(); if (parseExpected(SyntaxKind.OpenBraceToken)) { node.members = parseDelimitedList(ParsingContext.EnumMembers, parseEnumMember); parseExpected(SyntaxKind.CloseBraceToken); } else { node.members = createMissingList(); } return finishNode(node); } function parseModuleBlock(): ModuleBlock { const node = createNode(SyntaxKind.ModuleBlock); if (parseExpected(SyntaxKind.OpenBraceToken)) { node.statements = parseList(ParsingContext.BlockStatements, parseStatement); parseExpected(SyntaxKind.CloseBraceToken); } else { node.statements = createMissingList(); } return finishNode(node); } function parseModuleOrNamespaceDeclaration(node: ModuleDeclaration, flags: NodeFlags): ModuleDeclaration { node.kind = SyntaxKind.ModuleDeclaration; // If we are parsing a dotted namespace name, we want to // propagate the 'Namespace' flag across the names if set. const namespaceFlag = flags & NodeFlags.Namespace; node.flags |= flags; node.name = parseIdentifier(); node.body = parseOptional(SyntaxKind.DotToken) ? parseModuleOrNamespaceDeclaration(createNode(SyntaxKind.Unknown), NodeFlags.NestedNamespace | namespaceFlag) : parseModuleBlock(); return finishNode(node); } function parseAmbientExternalModuleDeclaration(node: ModuleDeclaration): ModuleDeclaration { node.kind = SyntaxKind.ModuleDeclaration; if (token() === SyntaxKind.GlobalKeyword) { // parse 'global' as name of global scope augmentation node.name = parseIdentifier(); node.flags |= NodeFlags.GlobalAugmentation; } else { node.name = parseLiteralNode(); node.name.text = internIdentifier(node.name.text); } if (token() === SyntaxKind.OpenBraceToken) { node.body = parseModuleBlock(); } else { parseSemicolon(); } return finishNode(node); } function parseModuleDeclaration(node: ModuleDeclaration): ModuleDeclaration { let flags: NodeFlags = 0; if (token() === SyntaxKind.GlobalKeyword) { // global augmentation return parseAmbientExternalModuleDeclaration(node); } else if (parseOptional(SyntaxKind.NamespaceKeyword)) { flags |= NodeFlags.Namespace; } else { parseExpected(SyntaxKind.ModuleKeyword); if (token() === SyntaxKind.StringLiteral) { return parseAmbientExternalModuleDeclaration(node); } } return parseModuleOrNamespaceDeclaration(node, flags); } function isExternalModuleReference() { return token() === SyntaxKind.RequireKeyword && lookAhead(nextTokenIsOpenParen); } function nextTokenIsOpenParen() { return nextToken() === SyntaxKind.OpenParenToken; } function nextTokenIsSlash() { return nextToken() === SyntaxKind.SlashToken; } function parseNamespaceExportDeclaration(node: NamespaceExportDeclaration): NamespaceExportDeclaration { node.kind = SyntaxKind.NamespaceExportDeclaration; parseExpected(SyntaxKind.AsKeyword); parseExpected(SyntaxKind.NamespaceKeyword); node.name = parseIdentifier(); parseSemicolon(); return finishNode(node); } function parseImportDeclarationOrImportEqualsDeclaration(node: ImportEqualsDeclaration | ImportDeclaration): ImportEqualsDeclaration | ImportDeclaration { parseExpected(SyntaxKind.ImportKeyword); const afterImportPos = scanner.getStartPos(); let identifier: Identifier; if (isIdentifier()) { identifier = parseIdentifier(); if (token() !== SyntaxKind.CommaToken && token() !== SyntaxKind.FromKeyword) { return parseImportEqualsDeclaration(node, identifier); } } // Import statement node.kind = SyntaxKind.ImportDeclaration; // ImportDeclaration: // import ImportClause from ModuleSpecifier ; // import ModuleSpecifier; if (identifier || // import id token() === SyntaxKind.AsteriskToken || // import * token() === SyntaxKind.OpenBraceToken) { // import { (node).importClause = parseImportClause(identifier, afterImportPos); parseExpected(SyntaxKind.FromKeyword); } (node).moduleSpecifier = parseModuleSpecifier(); parseSemicolon(); return finishNode(node); } function parseImportEqualsDeclaration(node: ImportEqualsDeclaration, identifier: ts.Identifier): ImportEqualsDeclaration { node.kind = SyntaxKind.ImportEqualsDeclaration; node.name = identifier; parseExpected(SyntaxKind.EqualsToken); node.moduleReference = parseModuleReference(); parseSemicolon(); return finishNode(node); } function parseImportClause(identifier: Identifier, fullStart: number) { // ImportClause: // ImportedDefaultBinding // NameSpaceImport // NamedImports // ImportedDefaultBinding, NameSpaceImport // ImportedDefaultBinding, NamedImports const importClause = createNode(SyntaxKind.ImportClause, fullStart); if (identifier) { // ImportedDefaultBinding: // ImportedBinding importClause.name = identifier; } // If there was no default import or if there is comma token after default import // parse namespace or named imports if (!importClause.name || parseOptional(SyntaxKind.CommaToken)) { importClause.namedBindings = token() === SyntaxKind.AsteriskToken ? parseNamespaceImport() : parseNamedImportsOrExports(SyntaxKind.NamedImports); } return finishNode(importClause); } function parseModuleReference() { return isExternalModuleReference() ? parseExternalModuleReference() : parseEntityName(/*allowReservedWords*/ false); } function parseExternalModuleReference() { const node = createNode(SyntaxKind.ExternalModuleReference); parseExpected(SyntaxKind.RequireKeyword); parseExpected(SyntaxKind.OpenParenToken); node.expression = parseModuleSpecifier(); parseExpected(SyntaxKind.CloseParenToken); return finishNode(node); } function parseModuleSpecifier(): Expression { if (token() === SyntaxKind.StringLiteral) { const result = parseLiteralNode(); result.text = internIdentifier(result.text); return result; } else { // We allow arbitrary expressions here, even though the grammar only allows string // literals. We check to ensure that it is only a string literal later in the grammar // check pass. return parseExpression(); } } function parseNamespaceImport(): NamespaceImport { // NameSpaceImport: // * as ImportedBinding const namespaceImport = createNode(SyntaxKind.NamespaceImport); parseExpected(SyntaxKind.AsteriskToken); parseExpected(SyntaxKind.AsKeyword); namespaceImport.name = parseIdentifier(); return finishNode(namespaceImport); } function parseNamedImportsOrExports(kind: SyntaxKind.NamedImports): NamedImports; function parseNamedImportsOrExports(kind: SyntaxKind.NamedExports): NamedExports; function parseNamedImportsOrExports(kind: SyntaxKind): NamedImportsOrExports { const node = createNode(kind); // NamedImports: // { } // { ImportsList } // { ImportsList, } // ImportsList: // ImportSpecifier // ImportsList, ImportSpecifier node.elements = | NodeArray>parseBracketedList(ParsingContext.ImportOrExportSpecifiers, kind === SyntaxKind.NamedImports ? parseImportSpecifier : parseExportSpecifier, SyntaxKind.OpenBraceToken, SyntaxKind.CloseBraceToken); return finishNode(node); } function parseExportSpecifier() { return parseImportOrExportSpecifier(SyntaxKind.ExportSpecifier); } function parseImportSpecifier() { return parseImportOrExportSpecifier(SyntaxKind.ImportSpecifier); } function parseImportOrExportSpecifier(kind: SyntaxKind): ImportOrExportSpecifier { const node = createNode(kind); // ImportSpecifier: // BindingIdentifier // IdentifierName as BindingIdentifier // ExportSpecifier: // IdentifierName // IdentifierName as IdentifierName let checkIdentifierIsKeyword = isKeyword(token()) && !isIdentifier(); let checkIdentifierStart = scanner.getTokenPos(); let checkIdentifierEnd = scanner.getTextPos(); const identifierName = parseIdentifierName(); if (token() === SyntaxKind.AsKeyword) { node.propertyName = identifierName; parseExpected(SyntaxKind.AsKeyword); checkIdentifierIsKeyword = isKeyword(token()) && !isIdentifier(); checkIdentifierStart = scanner.getTokenPos(); checkIdentifierEnd = scanner.getTextPos(); node.name = parseIdentifierName(); } else { node.name = identifierName; } if (kind === SyntaxKind.ImportSpecifier && checkIdentifierIsKeyword) { // Report error identifier expected parseErrorAtPosition(checkIdentifierStart, checkIdentifierEnd - checkIdentifierStart, Diagnostics.Identifier_expected); } return finishNode(node); } function parseExportDeclaration(node: ExportDeclaration): ExportDeclaration { node.kind = SyntaxKind.ExportDeclaration; if (parseOptional(SyntaxKind.AsteriskToken)) { parseExpected(SyntaxKind.FromKeyword); node.moduleSpecifier = parseModuleSpecifier(); } else { node.exportClause = parseNamedImportsOrExports(SyntaxKind.NamedExports); // It is not uncommon to accidentally omit the 'from' keyword. Additionally, in editing scenarios, // the 'from' keyword can be parsed as a named export when the export clause is unterminated (i.e. `export { from "moduleName";`) // If we don't have a 'from' keyword, see if we have a string literal such that ASI won't take effect. if (token() === SyntaxKind.FromKeyword || (token() === SyntaxKind.StringLiteral && !scanner.hasPrecedingLineBreak())) { parseExpected(SyntaxKind.FromKeyword); node.moduleSpecifier = parseModuleSpecifier(); } } parseSemicolon(); return finishNode(node); } function parseExportAssignment(node: ExportAssignment): ExportAssignment { node.kind = SyntaxKind.ExportAssignment; if (parseOptional(SyntaxKind.EqualsToken)) { node.isExportEquals = true; } else { parseExpected(SyntaxKind.DefaultKeyword); } node.expression = parseAssignmentExpressionOrHigher(); parseSemicolon(); return finishNode(node); } function processReferenceComments(sourceFile: SourceFile): void { const triviaScanner = createScanner(sourceFile.languageVersion, /*skipTrivia*/ false, LanguageVariant.Standard, sourceText); const referencedFiles: FileReference[] = []; const typeReferenceDirectives: FileReference[] = []; const amdDependencies: { path: string; name: string }[] = []; let amdModuleName: string; let checkJsDirective: CheckJsDirective = undefined; // Keep scanning all the leading trivia in the file until we get to something that // isn't trivia. Any single line comment will be analyzed to see if it is a // reference comment. while (true) { const kind = triviaScanner.scan(); if (kind !== SyntaxKind.SingleLineCommentTrivia) { if (isTrivia(kind)) { continue; } else { break; } } const range = { kind: triviaScanner.getToken(), pos: triviaScanner.getTokenPos(), end: triviaScanner.getTextPos(), }; const comment = sourceText.substring(range.pos, range.end); const referencePathMatchResult = getFileReferenceFromReferencePath(comment, range); if (referencePathMatchResult) { const fileReference = referencePathMatchResult.fileReference; sourceFile.hasNoDefaultLib = referencePathMatchResult.isNoDefaultLib; const diagnosticMessage = referencePathMatchResult.diagnosticMessage; if (fileReference) { if (referencePathMatchResult.isTypeReferenceDirective) { typeReferenceDirectives.push(fileReference); } else { referencedFiles.push(fileReference); } } if (diagnosticMessage) { parseDiagnostics.push(createFileDiagnostic(sourceFile, range.pos, range.end - range.pos, diagnosticMessage)); } } else { const amdModuleNameRegEx = /^\/\/\/\s* hasModifier(node, ModifierFlags.Export) || node.kind === SyntaxKind.ImportEqualsDeclaration && (node).moduleReference.kind === SyntaxKind.ExternalModuleReference || node.kind === SyntaxKind.ImportDeclaration || node.kind === SyntaxKind.ExportAssignment || node.kind === SyntaxKind.ExportDeclaration ? node : undefined); } const enum ParsingContext { SourceElements, // Elements in source file BlockStatements, // Statements in block SwitchClauses, // Clauses in switch statement SwitchClauseStatements, // Statements in switch clause TypeMembers, // Members in interface or type literal ClassMembers, // Members in class declaration EnumMembers, // Members in enum declaration HeritageClauseElement, // Elements in a heritage clause VariableDeclarations, // Variable declarations in variable statement ObjectBindingElements, // Binding elements in object binding list ArrayBindingElements, // Binding elements in array binding list ArgumentExpressions, // Expressions in argument list ObjectLiteralMembers, // Members in object literal JsxAttributes, // Attributes in jsx element JsxChildren, // Things between opening and closing JSX tags ArrayLiteralMembers, // Members in array literal Parameters, // Parameters in parameter list RestProperties, // Property names in a rest type list TypeParameters, // Type parameters in type parameter list TypeArguments, // Type arguments in type argument list TupleElementTypes, // Element types in tuple element type list HeritageClauses, // Heritage clauses for a class or interface declaration. ImportOrExportSpecifiers, // Named import clause's import specifier list Count // Number of parsing contexts } const enum Tristate { False, True, Unknown } export namespace JSDocParser { export function parseJSDocTypeExpressionForTests(content: string, start: number, length: number): { jsDocTypeExpression: JSDocTypeExpression, diagnostics: Diagnostic[] } | undefined { initializeState(content, ScriptTarget.Latest, /*_syntaxCursor:*/ undefined, ScriptKind.JS); sourceFile = createSourceFile("file.js", ScriptTarget.Latest, ScriptKind.JS, /*isDeclarationFile*/ false); scanner.setText(content, start, length); currentToken = scanner.scan(); const jsDocTypeExpression = parseJSDocTypeExpression(); const diagnostics = parseDiagnostics; clearState(); return jsDocTypeExpression ? { jsDocTypeExpression, diagnostics } : undefined; } // Parses out a JSDoc type expression. export function parseJSDocTypeExpression(mayOmitBraces?: boolean): JSDocTypeExpression { const result = createNode(SyntaxKind.JSDocTypeExpression, scanner.getTokenPos()); const hasBrace = (mayOmitBraces ? parseOptional : parseExpected)(SyntaxKind.OpenBraceToken); result.type = doInsideOfContext(NodeFlags.JSDoc, parseType); if (!mayOmitBraces || hasBrace) { parseExpected(SyntaxKind.CloseBraceToken); } fixupParentReferences(result); return finishNode(result); } export function parseIsolatedJSDocComment(content: string, start: number, length: number): { jsDoc: JSDoc, diagnostics: Diagnostic[] } | undefined { initializeState(content, ScriptTarget.Latest, /*_syntaxCursor:*/ undefined, ScriptKind.JS); sourceFile = { languageVariant: LanguageVariant.Standard, text: content }; // tslint:disable-line no-object-literal-type-assertion const jsDoc = parseJSDocCommentWorker(start, length); const diagnostics = parseDiagnostics; clearState(); return jsDoc ? { jsDoc, diagnostics } : undefined; } export function parseJSDocComment(parent: HasJSDoc, start: number, length: number): JSDoc { const saveToken = currentToken; const saveParseDiagnosticsLength = parseDiagnostics.length; const saveParseErrorBeforeNextFinishedNode = parseErrorBeforeNextFinishedNode; const comment = parseJSDocCommentWorker(start, length); if (comment) { comment.parent = parent; } if (contextFlags & NodeFlags.JavaScriptFile) { if (!sourceFile.jsDocDiagnostics) { sourceFile.jsDocDiagnostics = []; } sourceFile.jsDocDiagnostics.push(...parseDiagnostics); } currentToken = saveToken; parseDiagnostics.length = saveParseDiagnosticsLength; parseErrorBeforeNextFinishedNode = saveParseErrorBeforeNextFinishedNode; return comment; } const enum JSDocState { BeginningOfLine, SawAsterisk, SavingComments, } const enum PropertyLikeParse { Property, Parameter, } export function parseJSDocCommentWorker(start: number, length: number): JSDoc { const content = sourceText; start = start || 0; const end = length === undefined ? content.length : start + length; length = end - start; Debug.assert(start >= 0); Debug.assert(start <= end); Debug.assert(end <= content.length); let tags: JSDocTag[]; let tagsPos: number; let tagsEnd: number; const comments: string[] = []; let result: JSDoc; // Check for /** (JSDoc opening part) if (!isJsDocStart(content, start)) { return result; } // + 3 for leading /**, - 5 in total for /** */ scanner.scanRange(start + 3, length - 5, () => { // Initially we can parse out a tag. We also have seen a starting asterisk. // This is so that /** * @type */ doesn't parse. let state = JSDocState.SawAsterisk; let margin: number | undefined = undefined; // + 4 for leading '/** ' let indent = start - Math.max(content.lastIndexOf("\n", start), 0) + 4; function pushComment(text: string) { if (!margin) { margin = indent; } comments.push(text); indent += text.length; } let t = nextJSDocToken(); while (t === SyntaxKind.WhitespaceTrivia) { t = nextJSDocToken(); } if (t === SyntaxKind.NewLineTrivia) { state = JSDocState.BeginningOfLine; indent = 0; t = nextJSDocToken(); } loop: while (true) { switch (t) { case SyntaxKind.AtToken: if (state === JSDocState.BeginningOfLine || state === JSDocState.SawAsterisk) { removeTrailingNewlines(comments); parseTag(indent); // NOTE: According to usejsdoc.org, a tag goes to end of line, except the last tag. // Real-world comments may break this rule, so "BeginningOfLine" will not be a real line beginning // for malformed examples like `/** @param {string} x @returns {number} the length */` state = JSDocState.BeginningOfLine; margin = undefined; indent++; } else { pushComment(scanner.getTokenText()); } break; case SyntaxKind.NewLineTrivia: comments.push(scanner.getTokenText()); state = JSDocState.BeginningOfLine; indent = 0; break; case SyntaxKind.AsteriskToken: const asterisk = scanner.getTokenText(); if (state === JSDocState.SawAsterisk || state === JSDocState.SavingComments) { // If we've already seen an asterisk, then we can no longer parse a tag on this line state = JSDocState.SavingComments; pushComment(asterisk); } else { // Ignore the first asterisk on a line state = JSDocState.SawAsterisk; indent += asterisk.length; } break; case SyntaxKind.Identifier: // Anything else is doc comment text. We just save it. Because it // wasn't a tag, we can no longer parse a tag on this line until we hit the next // line break. pushComment(scanner.getTokenText()); state = JSDocState.SavingComments; break; case SyntaxKind.WhitespaceTrivia: // only collect whitespace if we're already saving comments or have just crossed the comment indent margin const whitespace = scanner.getTokenText(); if (state === JSDocState.SavingComments) { comments.push(whitespace); } else if (margin !== undefined && indent + whitespace.length > margin) { comments.push(whitespace.slice(margin - indent - 1)); } indent += whitespace.length; break; case SyntaxKind.EndOfFileToken: break loop; default: // anything other than whitespace or asterisk at the beginning of the line starts the comment text state = JSDocState.SavingComments; pushComment(scanner.getTokenText()); break; } t = nextJSDocToken(); } removeLeadingNewlines(comments); removeTrailingNewlines(comments); result = createJSDocComment(); }); return result; function removeLeadingNewlines(comments: string[]) { while (comments.length && (comments[0] === "\n" || comments[0] === "\r")) { comments.shift(); } } function removeTrailingNewlines(comments: string[]) { while (comments.length && (comments[comments.length - 1] === "\n" || comments[comments.length - 1] === "\r")) { comments.pop(); } } function isJsDocStart(content: string, start: number) { return content.charCodeAt(start) === CharacterCodes.slash && content.charCodeAt(start + 1) === CharacterCodes.asterisk && content.charCodeAt(start + 2) === CharacterCodes.asterisk && content.charCodeAt(start + 3) !== CharacterCodes.asterisk; } function createJSDocComment(): JSDoc { const result = createNode(SyntaxKind.JSDocComment, start); result.tags = tags && createNodeArray(tags, tagsPos, tagsEnd); result.comment = comments.length ? comments.join("") : undefined; return finishNode(result, end); } function skipWhitespace(): void { while (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) { nextJSDocToken(); } } function parseTag(indent: number) { Debug.assert(token() === SyntaxKind.AtToken); const atToken = createNode(SyntaxKind.AtToken, scanner.getTokenPos()); atToken.end = scanner.getTextPos(); nextJSDocToken(); const tagName = parseJSDocIdentifierName(); skipWhitespace(); if (!tagName) { return; } let tag: JSDocTag; if (tagName) { switch (tagName.escapedText) { case "augments": case "extends": tag = parseAugmentsTag(atToken, tagName); break; case "class": case "constructor": tag = parseClassTag(atToken, tagName); break; case "arg": case "argument": case "param": tag = parseParameterOrPropertyTag(atToken, tagName, PropertyLikeParse.Parameter); break; case "return": case "returns": tag = parseReturnTag(atToken, tagName); break; case "template": tag = parseTemplateTag(atToken, tagName); break; case "type": tag = parseTypeTag(atToken, tagName); break; case "typedef": tag = parseTypedefTag(atToken, tagName); break; default: tag = parseUnknownTag(atToken, tagName); break; } } else { tag = parseUnknownTag(atToken, tagName); } if (!tag) { // a badly malformed tag should not be added to the list of tags return; } tag.comment = parseTagComments(indent + tag.end - tag.pos); addTag(tag); } function parseTagComments(indent: number): string | undefined { const comments: string[] = []; let state = JSDocState.BeginningOfLine; let margin: number | undefined; function pushComment(text: string) { if (!margin) { margin = indent; } comments.push(text); indent += text.length; } let tok = token() as JsDocSyntaxKind; loop: while (true) { switch (tok) { case SyntaxKind.NewLineTrivia: if (state >= JSDocState.SawAsterisk) { state = JSDocState.BeginningOfLine; comments.push(scanner.getTokenText()); } indent = 0; break; case SyntaxKind.AtToken: scanner.setTextPos(scanner.getTextPos() - 1); // falls through case SyntaxKind.EndOfFileToken: // Done break loop; case SyntaxKind.WhitespaceTrivia: if (state === JSDocState.SavingComments) { pushComment(scanner.getTokenText()); } else { const whitespace = scanner.getTokenText(); // if the whitespace crosses the margin, take only the whitespace that passes the margin if (margin !== undefined && indent + whitespace.length > margin) { comments.push(whitespace.slice(margin - indent - 1)); } indent += whitespace.length; } break; case SyntaxKind.AsteriskToken: if (state === JSDocState.BeginningOfLine) { // leading asterisks start recording on the *next* (non-whitespace) token state = JSDocState.SawAsterisk; indent += 1; break; } // record the * as a comment // falls through default: state = JSDocState.SavingComments; // leading identifiers start recording as well pushComment(scanner.getTokenText()); break; } tok = nextJSDocToken(); } removeLeadingNewlines(comments); removeTrailingNewlines(comments); return comments.length === 0 ? undefined : comments.join(""); } function parseUnknownTag(atToken: AtToken, tagName: Identifier) { const result = createNode(SyntaxKind.JSDocTag, atToken.pos); result.atToken = atToken; result.tagName = tagName; return finishNode(result); } function addTag(tag: JSDocTag): void { if (!tags) { tags = [tag]; tagsPos = tag.pos; } else { tags.push(tag); } tagsEnd = tag.end; } function tryParseTypeExpression(): JSDocTypeExpression | undefined { skipWhitespace(); return token() === SyntaxKind.OpenBraceToken ? parseJSDocTypeExpression() : undefined; } function parseBracketNameInPropertyAndParamTag(): { name: EntityName, isBracketed: boolean } { // Looking for something like '[foo]', 'foo', '[foo.bar]' or 'foo.bar' const isBracketed = parseOptional(SyntaxKind.OpenBracketToken); const name = parseJSDocEntityName(); if (isBracketed) { skipWhitespace(); // May have an optional default, e.g. '[foo = 42]' if (parseOptionalToken(SyntaxKind.EqualsToken)) { parseExpression(); } parseExpected(SyntaxKind.CloseBracketToken); } return { name, isBracketed }; } function isObjectOrObjectArrayTypeReference(node: TypeNode): boolean { switch (node.kind) { case SyntaxKind.ObjectKeyword: return true; case SyntaxKind.ArrayType: return isObjectOrObjectArrayTypeReference((node as ArrayTypeNode).elementType); default: return isTypeReferenceNode(node) && ts.isIdentifier(node.typeName) && node.typeName.escapedText === "Object"; } } function parseParameterOrPropertyTag(atToken: AtToken, tagName: Identifier, target: PropertyLikeParse): JSDocParameterTag | JSDocPropertyTag { let typeExpression = tryParseTypeExpression(); let isNameFirst = !typeExpression; skipWhitespace(); const { name, isBracketed } = parseBracketNameInPropertyAndParamTag(); skipWhitespace(); if (isNameFirst) { typeExpression = tryParseTypeExpression(); } const result = target === PropertyLikeParse.Parameter ? createNode(SyntaxKind.JSDocParameterTag, atToken.pos) : createNode(SyntaxKind.JSDocPropertyTag, atToken.pos); const nestedTypeLiteral = parseNestedTypeLiteral(typeExpression, name); if (nestedTypeLiteral) { typeExpression = nestedTypeLiteral; isNameFirst = true; } result.atToken = atToken; result.tagName = tagName; result.typeExpression = typeExpression; result.name = name; result.isNameFirst = isNameFirst; result.isBracketed = isBracketed; return finishNode(result); } function parseNestedTypeLiteral(typeExpression: JSDocTypeExpression, name: EntityName) { if (typeExpression && isObjectOrObjectArrayTypeReference(typeExpression.type)) { const typeLiteralExpression = createNode(SyntaxKind.JSDocTypeExpression, scanner.getTokenPos()); let child: JSDocParameterTag | false; let jsdocTypeLiteral: JSDocTypeLiteral; const start = scanner.getStartPos(); let children: JSDocParameterTag[]; while (child = tryParse(() => parseChildParameterOrPropertyTag(PropertyLikeParse.Parameter, name))) { children = append(children, child); } if (children) { jsdocTypeLiteral = createNode(SyntaxKind.JSDocTypeLiteral, start); jsdocTypeLiteral.jsDocPropertyTags = children; if (typeExpression.type.kind === SyntaxKind.ArrayType) { jsdocTypeLiteral.isArrayType = true; } typeLiteralExpression.type = finishNode(jsdocTypeLiteral); return finishNode(typeLiteralExpression); } } } function parseReturnTag(atToken: AtToken, tagName: Identifier): JSDocReturnTag { if (forEach(tags, t => t.kind === SyntaxKind.JSDocReturnTag)) { parseErrorAtPosition(tagName.pos, scanner.getTokenPos() - tagName.pos, Diagnostics._0_tag_already_specified, tagName.escapedText); } const result = createNode(SyntaxKind.JSDocReturnTag, atToken.pos); result.atToken = atToken; result.tagName = tagName; result.typeExpression = tryParseTypeExpression(); return finishNode(result); } function parseTypeTag(atToken: AtToken, tagName: Identifier): JSDocTypeTag { if (forEach(tags, t => t.kind === SyntaxKind.JSDocTypeTag)) { parseErrorAtPosition(tagName.pos, scanner.getTokenPos() - tagName.pos, Diagnostics._0_tag_already_specified, tagName.escapedText); } const result = createNode(SyntaxKind.JSDocTypeTag, atToken.pos); result.atToken = atToken; result.tagName = tagName; result.typeExpression = parseJSDocTypeExpression(/*mayOmitBraces*/ true); return finishNode(result); } function parseAugmentsTag(atToken: AtToken, tagName: Identifier): JSDocAugmentsTag { const result = createNode(SyntaxKind.JSDocAugmentsTag, atToken.pos); result.atToken = atToken; result.tagName = tagName; result.class = parseExpressionWithTypeArgumentsForAugments(); return finishNode(result); } function parseExpressionWithTypeArgumentsForAugments(): ExpressionWithTypeArguments & { expression: Identifier | PropertyAccessEntityNameExpression } { const usedBrace = parseOptional(SyntaxKind.OpenBraceToken); const node = createNode(SyntaxKind.ExpressionWithTypeArguments) as ExpressionWithTypeArguments & { expression: Identifier | PropertyAccessEntityNameExpression }; node.expression = parsePropertyAccessEntityNameExpression(); node.typeArguments = tryParseTypeArguments(); const res = finishNode(node); if (usedBrace) { parseExpected(SyntaxKind.CloseBraceToken); } return res; } function parsePropertyAccessEntityNameExpression() { let node: Identifier | PropertyAccessEntityNameExpression = parseJSDocIdentifierName(/*createIfMissing*/ true); while (parseOptional(SyntaxKind.DotToken)) { const prop: PropertyAccessEntityNameExpression = createNode(SyntaxKind.PropertyAccessExpression, node.pos) as PropertyAccessEntityNameExpression; prop.expression = node; prop.name = parseJSDocIdentifierName(); node = finishNode(prop); } return node; } function parseClassTag(atToken: AtToken, tagName: Identifier): JSDocClassTag { const tag = createNode(SyntaxKind.JSDocClassTag, atToken.pos); tag.atToken = atToken; tag.tagName = tagName; return finishNode(tag); } function parseTypedefTag(atToken: AtToken, tagName: Identifier): JSDocTypedefTag { const typeExpression = tryParseTypeExpression(); skipWhitespace(); const typedefTag = createNode(SyntaxKind.JSDocTypedefTag, atToken.pos); typedefTag.atToken = atToken; typedefTag.tagName = tagName; typedefTag.fullName = parseJSDocTypeNameWithNamespace(/*flags*/ 0); if (typedefTag.fullName) { let rightNode = typedefTag.fullName; while (true) { if (rightNode.kind === SyntaxKind.Identifier || !rightNode.body) { // if node is identifier - use it as name // otherwise use name of the rightmost part that we were able to parse typedefTag.name = rightNode.kind === SyntaxKind.Identifier ? rightNode : rightNode.name; break; } rightNode = rightNode.body; } } skipWhitespace(); typedefTag.typeExpression = typeExpression; if (!typeExpression || isObjectOrObjectArrayTypeReference(typeExpression.type)) { let child: JSDocTypeTag | JSDocPropertyTag | false; let jsdocTypeLiteral: JSDocTypeLiteral; let childTypeTag: JSDocTypeTag; const start = scanner.getStartPos(); while (child = tryParse(() => parseChildParameterOrPropertyTag(PropertyLikeParse.Property))) { if (!jsdocTypeLiteral) { jsdocTypeLiteral = createNode(SyntaxKind.JSDocTypeLiteral, start); } if (child.kind === SyntaxKind.JSDocTypeTag) { if (childTypeTag) { break; } else { childTypeTag = child; } } else { jsdocTypeLiteral.jsDocPropertyTags = append(jsdocTypeLiteral.jsDocPropertyTags as MutableNodeArray, child); } } if (jsdocTypeLiteral) { if (typeExpression && typeExpression.type.kind === SyntaxKind.ArrayType) { jsdocTypeLiteral.isArrayType = true; } typedefTag.typeExpression = childTypeTag && childTypeTag.typeExpression && !isObjectOrObjectArrayTypeReference(childTypeTag.typeExpression.type) ? childTypeTag.typeExpression : finishNode(jsdocTypeLiteral); } } return finishNode(typedefTag); function parseJSDocTypeNameWithNamespace(flags: NodeFlags) { const pos = scanner.getTokenPos(); const typeNameOrNamespaceName = parseJSDocIdentifierName(); if (typeNameOrNamespaceName && parseOptional(SyntaxKind.DotToken)) { const jsDocNamespaceNode = createNode(SyntaxKind.ModuleDeclaration, pos); jsDocNamespaceNode.flags |= flags; jsDocNamespaceNode.name = typeNameOrNamespaceName; jsDocNamespaceNode.body = parseJSDocTypeNameWithNamespace(NodeFlags.NestedNamespace); return finishNode(jsDocNamespaceNode); } if (typeNameOrNamespaceName && flags & NodeFlags.NestedNamespace) { typeNameOrNamespaceName.isInJSDocNamespace = true; } return typeNameOrNamespaceName; } } function escapedTextsEqual(a: EntityName, b: EntityName): boolean { while (!ts.isIdentifier(a) || !ts.isIdentifier(b)) { if (!ts.isIdentifier(a) && !ts.isIdentifier(b) && a.right.escapedText === b.right.escapedText) { a = a.left; b = b.left; } else { return false; } } return a.escapedText === b.escapedText; } function parseChildParameterOrPropertyTag(target: PropertyLikeParse.Property): JSDocTypeTag | JSDocPropertyTag | false; function parseChildParameterOrPropertyTag(target: PropertyLikeParse.Parameter, name: EntityName): JSDocParameterTag | false; function parseChildParameterOrPropertyTag(target: PropertyLikeParse, name?: EntityName): JSDocTypeTag | JSDocPropertyTag | JSDocParameterTag | false { let canParseTag = true; let seenAsterisk = false; while (true) { switch (nextJSDocToken()) { case SyntaxKind.AtToken: if (canParseTag) { const child = tryParseChildTag(target); if (child && child.kind === SyntaxKind.JSDocParameterTag && (ts.isIdentifier(child.name) || !escapedTextsEqual(name, child.name.left))) { return false; } return child; } seenAsterisk = false; break; case SyntaxKind.NewLineTrivia: canParseTag = true; seenAsterisk = false; break; case SyntaxKind.AsteriskToken: if (seenAsterisk) { canParseTag = false; } seenAsterisk = true; break; case SyntaxKind.Identifier: canParseTag = false; break; case SyntaxKind.EndOfFileToken: return false; } } } function tryParseChildTag(target: PropertyLikeParse): JSDocTypeTag | JSDocPropertyTag | JSDocParameterTag | false { Debug.assert(token() === SyntaxKind.AtToken); const atToken = createNode(SyntaxKind.AtToken); atToken.end = scanner.getTextPos(); nextJSDocToken(); const tagName = parseJSDocIdentifierName(); skipWhitespace(); if (!tagName) { return false; } let t: PropertyLikeParse; switch (tagName.escapedText) { case "type": return target === PropertyLikeParse.Property && parseTypeTag(atToken, tagName); case "prop": case "property": t = PropertyLikeParse.Property; break; case "arg": case "argument": case "param": t = PropertyLikeParse.Parameter; break; default: return false; } if (target !== t) { return false; } const tag = parseParameterOrPropertyTag(atToken, tagName, target); tag.comment = parseTagComments(tag.end - tag.pos); return tag; } function parseTemplateTag(atToken: AtToken, tagName: Identifier): JSDocTemplateTag | undefined { if (some(tags, isJSDocTemplateTag)) { parseErrorAtPosition(tagName.pos, scanner.getTokenPos() - tagName.pos, Diagnostics._0_tag_already_specified, tagName.escapedText); } // Type parameter list looks like '@template T,U,V' const typeParameters = []; const typeParametersPos = getNodePos(); while (true) { const typeParameter = createNode(SyntaxKind.TypeParameter); const name = parseJSDocIdentifierNameWithOptionalBraces(); skipWhitespace(); if (!name) { parseErrorAtPosition(scanner.getStartPos(), 0, Diagnostics.Identifier_expected); return undefined; } typeParameter.name = name; finishNode(typeParameter); typeParameters.push(typeParameter); if (token() === SyntaxKind.CommaToken) { nextJSDocToken(); skipWhitespace(); } else { break; } } const result = createNode(SyntaxKind.JSDocTemplateTag, atToken.pos); result.atToken = atToken; result.tagName = tagName; result.typeParameters = createNodeArray(typeParameters, typeParametersPos); finishNode(result); return result; } function parseJSDocIdentifierNameWithOptionalBraces(): Identifier | undefined { const parsedBrace = parseOptional(SyntaxKind.OpenBraceToken); const res = parseJSDocIdentifierName(); if (parsedBrace) { parseExpected(SyntaxKind.CloseBraceToken); } return res; } function nextJSDocToken(): JsDocSyntaxKind { return currentToken = scanner.scanJSDocToken(); } function parseJSDocEntityName(): EntityName { let entity: EntityName = parseJSDocIdentifierName(/*createIfMissing*/ true); if (parseOptional(SyntaxKind.OpenBracketToken)) { parseExpected(SyntaxKind.CloseBracketToken); // Note that y[] is accepted as an entity name, but the postfix brackets are not saved for checking. // Technically usejsdoc.org requires them for specifying a property of a type equivalent to Array<{ x: ...}> // but it's not worth it to enforce that restriction. } while (parseOptional(SyntaxKind.DotToken)) { const name = parseJSDocIdentifierName(/*createIfMissing*/ true); if (parseOptional(SyntaxKind.OpenBracketToken)) { parseExpected(SyntaxKind.CloseBracketToken); } entity = createQualifiedName(entity, name); } return entity; } function parseJSDocIdentifierName(): Identifier | undefined; function parseJSDocIdentifierName(createIfMissing: true): Identifier; function parseJSDocIdentifierName(createIfMissing = false): Identifier | undefined { if (!tokenIsIdentifierOrKeyword(token())) { if (createIfMissing) { return createMissingNode(SyntaxKind.Identifier, /*reportAtCurrentPosition*/ true, Diagnostics.Identifier_expected); } else { parseErrorAtCurrentToken(Diagnostics.Identifier_expected); return undefined; } } const pos = scanner.getTokenPos(); const end = scanner.getTextPos(); const result = createNode(SyntaxKind.Identifier, pos); result.escapedText = escapeLeadingUnderscores(content.substring(pos, end)); finishNode(result, end); nextJSDocToken(); return result; } } } } namespace IncrementalParser { export function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks: boolean): SourceFile { aggressiveChecks = aggressiveChecks || Debug.shouldAssert(AssertionLevel.Aggressive); checkChangeRange(sourceFile, newText, textChangeRange, aggressiveChecks); if (textChangeRangeIsUnchanged(textChangeRange)) { // if the text didn't change, then we can just return our current source file as-is. return sourceFile; } if (sourceFile.statements.length === 0) { // If we don't have any statements in the current source file, then there's no real // way to incrementally parse. So just do a full parse instead. return Parser.parseSourceFile(sourceFile.fileName, newText, sourceFile.languageVersion, /*syntaxCursor*/ undefined, /*setParentNodes*/ true, sourceFile.scriptKind); } // Make sure we're not trying to incrementally update a source file more than once. Once // we do an update the original source file is considered unusable from that point onwards. // // This is because we do incremental parsing in-place. i.e. we take nodes from the old // tree and give them new positions and parents. From that point on, trusting the old // tree at all is not possible as far too much of it may violate invariants. const incrementalSourceFile = sourceFile; Debug.assert(!incrementalSourceFile.hasBeenIncrementallyParsed); incrementalSourceFile.hasBeenIncrementallyParsed = true; const oldText = sourceFile.text; const syntaxCursor = createSyntaxCursor(sourceFile); // Make the actual change larger so that we know to reparse anything whose lookahead // might have intersected the change. const changeRange = extendToAffectedRange(sourceFile, textChangeRange); checkChangeRange(sourceFile, newText, changeRange, aggressiveChecks); // Ensure that extending the affected range only moved the start of the change range // earlier in the file. Debug.assert(changeRange.span.start <= textChangeRange.span.start); Debug.assert(textSpanEnd(changeRange.span) === textSpanEnd(textChangeRange.span)); Debug.assert(textSpanEnd(textChangeRangeNewSpan(changeRange)) === textSpanEnd(textChangeRangeNewSpan(textChangeRange))); // The is the amount the nodes after the edit range need to be adjusted. It can be // positive (if the edit added characters), negative (if the edit deleted characters) // or zero (if this was a pure overwrite with nothing added/removed). const delta = textChangeRangeNewSpan(changeRange).length - changeRange.span.length; // If we added or removed characters during the edit, then we need to go and adjust all // the nodes after the edit. Those nodes may move forward (if we inserted chars) or they // may move backward (if we deleted chars). // // Doing this helps us out in two ways. First, it means that any nodes/tokens we want // to reuse are already at the appropriate position in the new text. That way when we // reuse them, we don't have to figure out if they need to be adjusted. Second, it makes // it very easy to determine if we can reuse a node. If the node's position is at where // we are in the text, then we can reuse it. Otherwise we can't. If the node's position // is ahead of us, then we'll need to rescan tokens. If the node's position is behind // us, then we'll need to skip it or crumble it as appropriate // // We will also adjust the positions of nodes that intersect the change range as well. // By doing this, we ensure that all the positions in the old tree are consistent, not // just the positions of nodes entirely before/after the change range. By being // consistent, we can then easily map from positions to nodes in the old tree easily. // // Also, mark any syntax elements that intersect the changed span. We know, up front, // that we cannot reuse these elements. updateTokenPositionsAndMarkElements(incrementalSourceFile, changeRange.span.start, textSpanEnd(changeRange.span), textSpanEnd(textChangeRangeNewSpan(changeRange)), delta, oldText, newText, aggressiveChecks); // Now that we've set up our internal incremental state just proceed and parse the // source file in the normal fashion. When possible the parser will retrieve and // reuse nodes from the old tree. // // Note: passing in 'true' for setNodeParents is very important. When incrementally // parsing, we will be reusing nodes from the old tree, and placing it into new // parents. If we don't set the parents now, we'll end up with an observably // inconsistent tree. Setting the parents on the new tree should be very fast. We // will immediately bail out of walking any subtrees when we can see that their parents // are already correct. const result = Parser.parseSourceFile(sourceFile.fileName, newText, sourceFile.languageVersion, syntaxCursor, /*setParentNodes*/ true, sourceFile.scriptKind); return result; } function moveElementEntirelyPastChangeRange(element: IncrementalElement, isArray: boolean, delta: number, oldText: string, newText: string, aggressiveChecks: boolean) { if (isArray) { visitArray(element); } else { visitNode(element); } return; function visitNode(node: IncrementalNode) { let text = ""; if (aggressiveChecks && shouldCheckNode(node)) { text = oldText.substring(node.pos, node.end); } // Ditch any existing LS children we may have created. This way we can avoid // moving them forward. if (node._children) { node._children = undefined; } node.pos += delta; node.end += delta; if (aggressiveChecks && shouldCheckNode(node)) { Debug.assert(text === newText.substring(node.pos, node.end)); } forEachChild(node, visitNode, visitArray); if (hasJSDocNodes(node)) { for (const jsDocComment of node.jsDoc) { forEachChild(jsDocComment, visitNode, visitArray); } } checkNodePositions(node, aggressiveChecks); } function visitArray(array: IncrementalNodeArray) { array._children = undefined; array.pos += delta; array.end += delta; for (const node of array) { visitNode(node); } } } function shouldCheckNode(node: Node) { switch (node.kind) { case SyntaxKind.StringLiteral: case SyntaxKind.NumericLiteral: case SyntaxKind.Identifier: return true; } return false; } function adjustIntersectingElement(element: IncrementalElement, changeStart: number, changeRangeOldEnd: number, changeRangeNewEnd: number, delta: number) { Debug.assert(element.end >= changeStart, "Adjusting an element that was entirely before the change range"); Debug.assert(element.pos <= changeRangeOldEnd, "Adjusting an element that was entirely after the change range"); Debug.assert(element.pos <= element.end); // We have an element that intersects the change range in some way. It may have its // start, or its end (or both) in the changed range. We want to adjust any part // that intersects such that the final tree is in a consistent state. i.e. all // children have spans within the span of their parent, and all siblings are ordered // properly. // We may need to update both the 'pos' and the 'end' of the element. // If the 'pos' is before the start of the change, then we don't need to touch it. // If it isn't, then the 'pos' must be inside the change. How we update it will // depend if delta is positive or negative. If delta is positive then we have // something like: // // -------------------AAA----------------- // -------------------BBBCCCCCCC----------------- // // In this case, we consider any node that started in the change range to still be // starting at the same position. // // however, if the delta is negative, then we instead have something like this: // // -------------------XXXYYYYYYY----------------- // -------------------ZZZ----------------- // // In this case, any element that started in the 'X' range will keep its position. // However any element that started after that will have their pos adjusted to be // at the end of the new range. i.e. any node that started in the 'Y' range will // be adjusted to have their start at the end of the 'Z' range. // // The element will keep its position if possible. Or Move backward to the new-end // if it's in the 'Y' range. element.pos = Math.min(element.pos, changeRangeNewEnd); // If the 'end' is after the change range, then we always adjust it by the delta // amount. However, if the end is in the change range, then how we adjust it // will depend on if delta is positive or negative. If delta is positive then we // have something like: // // -------------------AAA----------------- // -------------------BBBCCCCCCC----------------- // // In this case, we consider any node that ended inside the change range to keep its // end position. // // however, if the delta is negative, then we instead have something like this: // // -------------------XXXYYYYYYY----------------- // -------------------ZZZ----------------- // // In this case, any element that ended in the 'X' range will keep its position. // However any element that ended after that will have their pos adjusted to be // at the end of the new range. i.e. any node that ended in the 'Y' range will // be adjusted to have their end at the end of the 'Z' range. if (element.end >= changeRangeOldEnd) { // Element ends after the change range. Always adjust the end pos. element.end += delta; } else { // Element ends in the change range. The element will keep its position if // possible. Or Move backward to the new-end if it's in the 'Y' range. element.end = Math.min(element.end, changeRangeNewEnd); } Debug.assert(element.pos <= element.end); if (element.parent) { Debug.assert(element.pos >= element.parent.pos); Debug.assert(element.end <= element.parent.end); } } function checkNodePositions(node: Node, aggressiveChecks: boolean) { if (aggressiveChecks) { let pos = node.pos; forEachChild(node, child => { Debug.assert(child.pos >= pos); pos = child.end; }); Debug.assert(pos <= node.end); } } function updateTokenPositionsAndMarkElements( sourceFile: IncrementalNode, changeStart: number, changeRangeOldEnd: number, changeRangeNewEnd: number, delta: number, oldText: string, newText: string, aggressiveChecks: boolean): void { visitNode(sourceFile); return; function visitNode(child: IncrementalNode) { Debug.assert(child.pos <= child.end); if (child.pos > changeRangeOldEnd) { // Node is entirely past the change range. We need to move both its pos and // end, forward or backward appropriately. moveElementEntirelyPastChangeRange(child, /*isArray*/ false, delta, oldText, newText, aggressiveChecks); return; } // Check if the element intersects the change range. If it does, then it is not // reusable. Also, we'll need to recurse to see what constituent portions we may // be able to use. const fullEnd = child.end; if (fullEnd >= changeStart) { child.intersectsChange = true; child._children = undefined; // Adjust the pos or end (or both) of the intersecting element accordingly. adjustIntersectingElement(child, changeStart, changeRangeOldEnd, changeRangeNewEnd, delta); forEachChild(child, visitNode, visitArray); checkNodePositions(child, aggressiveChecks); return; } // Otherwise, the node is entirely before the change range. No need to do anything with it. Debug.assert(fullEnd < changeStart); } function visitArray(array: IncrementalNodeArray) { Debug.assert(array.pos <= array.end); if (array.pos > changeRangeOldEnd) { // Array is entirely after the change range. We need to move it, and move any of // its children. moveElementEntirelyPastChangeRange(array, /*isArray*/ true, delta, oldText, newText, aggressiveChecks); return; } // Check if the element intersects the change range. If it does, then it is not // reusable. Also, we'll need to recurse to see what constituent portions we may // be able to use. const fullEnd = array.end; if (fullEnd >= changeStart) { array.intersectsChange = true; array._children = undefined; // Adjust the pos or end (or both) of the intersecting array accordingly. adjustIntersectingElement(array, changeStart, changeRangeOldEnd, changeRangeNewEnd, delta); for (const node of array) { visitNode(node); } return; } // Otherwise, the array is entirely before the change range. No need to do anything with it. Debug.assert(fullEnd < changeStart); } } function extendToAffectedRange(sourceFile: SourceFile, changeRange: TextChangeRange): TextChangeRange { // Consider the following code: // void foo() { /; } // // If the text changes with an insertion of / just before the semicolon then we end up with: // void foo() { //; } // // If we were to just use the changeRange a is, then we would not rescan the { token // (as it does not intersect the actual original change range). Because an edit may // change the token touching it, we actually need to look back *at least* one token so // that the prior token sees that change. const maxLookahead = 1; let start = changeRange.span.start; // the first iteration aligns us with the change start. subsequent iteration move us to // the left by maxLookahead tokens. We only need to do this as long as we're not at the // start of the tree. for (let i = 0; start > 0 && i <= maxLookahead; i++) { const nearestNode = findNearestNodeStartingBeforeOrAtPosition(sourceFile, start); Debug.assert(nearestNode.pos <= start); const position = nearestNode.pos; start = Math.max(0, position - 1); } const finalSpan = createTextSpanFromBounds(start, textSpanEnd(changeRange.span)); const finalLength = changeRange.newLength + (changeRange.span.start - start); return createTextChangeRange(finalSpan, finalLength); } function findNearestNodeStartingBeforeOrAtPosition(sourceFile: SourceFile, position: number): Node { let bestResult: Node = sourceFile; let lastNodeEntirelyBeforePosition: Node; forEachChild(sourceFile, visit); if (lastNodeEntirelyBeforePosition) { const lastChildOfLastEntireNodeBeforePosition = getLastChild(lastNodeEntirelyBeforePosition); if (lastChildOfLastEntireNodeBeforePosition.pos > bestResult.pos) { bestResult = lastChildOfLastEntireNodeBeforePosition; } } return bestResult; function getLastChild(node: Node): Node { while (true) { const lastChild = getLastChildWorker(node); if (lastChild) { node = lastChild; } else { return node; } } } function getLastChildWorker(node: Node): Node | undefined { let last: Node = undefined; forEachChild(node, child => { if (nodeIsPresent(child)) { last = child; } }); return last; } function visit(child: Node) { if (nodeIsMissing(child)) { // Missing nodes are effectively invisible to us. We never even consider them // When trying to find the nearest node before us. return; } // If the child intersects this position, then this node is currently the nearest // node that starts before the position. if (child.pos <= position) { if (child.pos >= bestResult.pos) { // This node starts before the position, and is closer to the position than // the previous best node we found. It is now the new best node. bestResult = child; } // Now, the node may overlap the position, or it may end entirely before the // position. If it overlaps with the position, then either it, or one of its // children must be the nearest node before the position. So we can just // recurse into this child to see if we can find something better. if (position < child.end) { // The nearest node is either this child, or one of the children inside // of it. We've already marked this child as the best so far. Recurse // in case one of the children is better. forEachChild(child, visit); // Once we look at the children of this node, then there's no need to // continue any further. return true; } else { Debug.assert(child.end <= position); // The child ends entirely before this position. Say you have the following // (where $ is the position) // // ? $ : <...> <...> // // We would want to find the nearest preceding node in "complex expr 2". // To support that, we keep track of this node, and once we're done searching // for a best node, we recurse down this node to see if we can find a good // result in it. // // This approach allows us to quickly skip over nodes that are entirely // before the position, while still allowing us to find any nodes in the // last one that might be what we want. lastNodeEntirelyBeforePosition = child; } } else { Debug.assert(child.pos > position); // We're now at a node that is entirely past the position we're searching for. // This node (and all following nodes) could never contribute to the result, // so just skip them by returning 'true' here. return true; } } } function checkChangeRange(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks: boolean) { const oldText = sourceFile.text; if (textChangeRange) { Debug.assert((oldText.length - textChangeRange.span.length + textChangeRange.newLength) === newText.length); if (aggressiveChecks || Debug.shouldAssert(AssertionLevel.VeryAggressive)) { const oldTextPrefix = oldText.substr(0, textChangeRange.span.start); const newTextPrefix = newText.substr(0, textChangeRange.span.start); Debug.assert(oldTextPrefix === newTextPrefix); const oldTextSuffix = oldText.substring(textSpanEnd(textChangeRange.span), oldText.length); const newTextSuffix = newText.substring(textSpanEnd(textChangeRangeNewSpan(textChangeRange)), newText.length); Debug.assert(oldTextSuffix === newTextSuffix); } } } interface IncrementalElement extends TextRange { parent?: Node; intersectsChange: boolean; length?: number; _children: Node[]; } export interface IncrementalNode extends Node, IncrementalElement { hasBeenIncrementallyParsed: boolean; } interface IncrementalNodeArray extends NodeArray, IncrementalElement { length: number; } // Allows finding nodes in the source file at a certain position in an efficient manner. // The implementation takes advantage of the calling pattern it knows the parser will // make in order to optimize finding nodes as quickly as possible. export interface SyntaxCursor { currentNode(position: number): IncrementalNode; } function createSyntaxCursor(sourceFile: SourceFile): SyntaxCursor { let currentArray: NodeArray = sourceFile.statements; let currentArrayIndex = 0; Debug.assert(currentArrayIndex < currentArray.length); let current = currentArray[currentArrayIndex]; let lastQueriedPosition = InvalidPosition.Value; return { currentNode(position: number) { // Only compute the current node if the position is different than the last time // we were asked. The parser commonly asks for the node at the same position // twice. Once to know if can read an appropriate list element at a certain point, // and then to actually read and consume the node. if (position !== lastQueriedPosition) { // Much of the time the parser will need the very next node in the array that // we just returned a node from.So just simply check for that case and move // forward in the array instead of searching for the node again. if (current && current.end === position && currentArrayIndex < (currentArray.length - 1)) { currentArrayIndex++; current = currentArray[currentArrayIndex]; } // If we don't have a node, or the node we have isn't in the right position, // then try to find a viable node at the position requested. if (!current || current.pos !== position) { findHighestListElementThatStartsAtPosition(position); } } // Cache this query so that we don't do any extra work if the parser calls back // into us. Note: this is very common as the parser will make pairs of calls like // 'isListElement -> parseListElement'. If we were unable to find a node when // called with 'isListElement', we don't want to redo the work when parseListElement // is called immediately after. lastQueriedPosition = position; // Either we don'd have a node, or we have a node at the position being asked for. Debug.assert(!current || current.pos === position); return current; } }; // Finds the highest element in the tree we can find that starts at the provided position. // The element must be a direct child of some node list in the tree. This way after we // return it, we can easily return its next sibling in the list. function findHighestListElementThatStartsAtPosition(position: number) { // Clear out any cached state about the last node we found. currentArray = undefined; currentArrayIndex = InvalidPosition.Value; current = undefined; // Recurse into the source file to find the highest node at this position. forEachChild(sourceFile, visitNode, visitArray); return; function visitNode(node: Node) { if (position >= node.pos && position < node.end) { // Position was within this node. Keep searching deeper to find the node. forEachChild(node, visitNode, visitArray); // don't proceed any further in the search. return true; } // position wasn't in this node, have to keep searching. return false; } function visitArray(array: NodeArray) { if (position >= array.pos && position < array.end) { // position was in this array. Search through this array to see if we find a // viable element. for (let i = 0; i < array.length; i++) { const child = array[i]; if (child) { if (child.pos === position) { // Found the right node. We're done. currentArray = array; currentArrayIndex = i; current = child; return true; } else { if (child.pos < position && position < child.end) { // Position in somewhere within this child. Search in it and // stop searching in this array. forEachChild(child, visitNode, visitArray); return true; } } } } } // position wasn't in this array, have to keep searching. return false; } } } const enum InvalidPosition { Value = -1 } } function isDeclarationFileName(fileName: string): boolean { return fileExtensionIs(fileName, Extension.Dts); } } ================================================ FILE: examples/typescript/small.ts ================================================ class Foo { constructor() {} } function foo() { } const s = `${foo()}` ================================================ FILE: package.json ================================================ { "name": "vscode-tree-sitter", "displayName": "Tree Sitter [Deprecated]", "description": "Accurate syntax coloring with tree-sitter", "icon": "tree-sitter-small.png", "version": "0.1.26", "preview": true, "publisher": "georgewfraser", "repository": { "type": "git", "url": "https://github.com/georgewfraser/vscode-tree-sitter" }, "license": "MIT", "extensionKind": [ "ui" ], "engines": { "vscode": "^1.34.0" }, "categories": [ "Programming Languages", "Themes", "Other" ], "activationEvents": [ "onLanguage:go", "onLanguage:cpp", "onLanguage:rust", "onLanguage:ruby", "onLanguage:typescript", "onLanguage:javascript" ], "main": "./out/extension.js", "contributes": { "grammars": [ { "language": "go", "scopeName": "source.go", "path": "./textmate/go.tmLanguage.json" }, { "language": "cpp", "scopeName": "source.cpp", "path": "./textmate/cpp.tmLanguage.json" }, { "language": "ruby", "scopeName": "source.ruby", "path": "./textmate/ruby.tmLanguage.json" }, { "language": "rust", "scopeName": "source.rust", "path": "./textmate/rust.tmLanguage.json" }, { "language": "typescript", "scopeName": "source.ts", "path": "./textmate/typescript.tmLanguage.json" }, { "language": "javascript", "scopeName": "source.ts", "path": "./textmate/typescript.tmLanguage.json" } ] }, "scripts": { "vscode:prepublish": "npm run compile", "compile": "tsc -p ./", "watch": "tsc -watch -p ./", "postinstall": "node ./node_modules/vscode/bin/install", "test": "npm run compile && node ./out/test", "benchmark": "npm run compile && node ./out/benchmark", "debug": "npm run compile && node --nolazy --inspect-brk=9229 ./out/test", "build": "vsce package -o build.vsix", "publish": "vsce publish patch" }, "devDependencies": { "@types/mocha": "^2.2.42", "@types/node": "^8.10.25", "tree-sitter-cli": "^0.16.5", "tree-sitter-cpp": "^0.16.0", "tree-sitter-go": "^0.16.0", "tree-sitter-javascript": "^0.16.0", "tree-sitter-ruby": "^0.16.1", "tree-sitter-rust": "^0.16.0", "tree-sitter-typescript": "^0.16.1", "tslint": "^6.0.0", "typescript": "^3.8.2", "vsce": "^1.73.0", "vscode": "^1.1.36" }, "dependencies": { "jsonc-parser": "^2.1.0", "tar": ">=4.4.2", "web-tree-sitter": "^0.16.2" } } ================================================ FILE: scripts/build.sh ================================================ #!/bin/bash set -e # Build vsix npm run-script build code --install-extension build.vsix --force echo 'Reload VSCode to update extension' ================================================ FILE: scripts/gen-parsers.sh ================================================ #!/usr/bin/env bash # TODO this still doesn't work on my mac laptop :( # fix it and delete parsers/*.wasm from git set -e # Build parsers ./node_modules/.bin/tree-sitter build-wasm ./node_modules/tree-sitter-go ./node_modules/.bin/tree-sitter build-wasm ./node_modules/tree-sitter-cpp ./node_modules/.bin/tree-sitter build-wasm ./node_modules/tree-sitter-ruby ./node_modules/.bin/tree-sitter build-wasm ./node_modules/tree-sitter-rust ./node_modules/.bin/tree-sitter build-wasm ./node_modules/tree-sitter-typescript/typescript ./node_modules/.bin/tree-sitter build-wasm ./node_modules/tree-sitter-javascript mv *.wasm parsers ================================================ FILE: src/benchmark.ts ================================================ // import extension = require('./extension') import Parser = require('web-tree-sitter') import fs = require('fs') import colors = require('./colors') benchmarkGo() async function benchmarkGo() { await Parser.init() const parser = new Parser() const wasm = 'parsers/tree-sitter-go.wasm' const lang = await Parser.Language.load(wasm) parser.setLanguage(lang) const text = fs.readFileSync('examples/go/proc.go', {encoding: 'utf-8'}) const tree = parser.parse(text) for (let i = 0; i < 10; i++) { console.time('colorGo') colors.colorGo(tree, [{start: 0, end: tree.rootNode.endPosition.row}]) console.timeEnd('colorGo') } } ================================================ FILE: src/colors.ts ================================================ import * as Parser from 'web-tree-sitter' export type Range = {start: Parser.Point, end: Parser.Point} export type ColorFunction = (x: Parser.Tree, visibleRanges: {start: number, end: number}[]) => Map export function colorGo(root: Parser.Tree, visibleRanges: {start: number, end: number}[]) { const functions: Range[] = [] const types: Range[] = [] const variables: Range[] = [] const underlines: Range[] = [] // Guess package names based on paths var packages: {[id: string]: boolean} = {} function scanImport(x: Parser.SyntaxNode) { if (x.type == 'import_spec') { let str = x.firstChild!.text if (str.startsWith('"')) { str = str.substring(1, str.length - 1) } const parts = str.split('/') const last = parts[parts.length - 1] packages[last] = true } for (const child of x.children) { scanImport(child) } } // Keep track of local vars that shadow packages const allScopes: Scope[] = [] class Scope { private locals = new Map() private parent: Scope|null constructor(parent: Scope|null) { this.parent = parent allScopes.push(this) } declareLocal(id: string) { if (this.isRoot()) return if (this.locals.has(id)) { this.locals.get(id)!.modified = true } else { this.locals.set(id, {modified: false, references: []}) } } modifyLocal(id: string) { if (this.isRoot()) return if (this.locals.has(id)) this.locals.get(id)!.modified = true else if (this.parent) this.parent.modifyLocal(id) } referenceLocal(x: Parser.SyntaxNode) { if (this.isRoot()) return const id = x.text if (this.locals.has(id)) this.locals.get(id)!.references.push(x) else if (this.parent) this.parent.referenceLocal(x) } isLocal(id: string): boolean { if (this.locals.has(id)) return true if (this.parent) return this.parent.isLocal(id) return false } isUnknown(id: string): boolean { if (packages[id]) return false if (this.locals.has(id)) return false if (this.parent) return this.parent.isUnknown(id) return true } isModified(id: string): boolean { if (this.locals.has(id)) return this.locals.get(id)!.modified if (this.parent) return this.parent.isModified(id) return false } modifiedLocals(): Parser.SyntaxNode[] { const all = [] for (const {modified, references} of this.locals.values()) { if (modified) { all.push(...references) } } return all } isPackage(id: string): boolean { return packages[id] && !this.isLocal(id) } isRoot(): boolean { return this.parent == null } } const rootScope = new Scope(null) function scanSourceFile() { for (const top of root.rootNode.namedChildren) { scanTopLevelDeclaration(top) } } function scanTopLevelDeclaration(x: Parser.SyntaxNode) { switch (x.type) { case 'import_declaration': scanImport(x) break case 'function_declaration': case 'method_declaration': if (!isVisible(x, visibleRanges)) return scanFunctionDeclaration(x) break case 'const_declaration': case 'var_declaration': if (!isVisible(x, visibleRanges)) return scanVarDeclaration(x) break case 'type_declaration': if (!isVisible(x, visibleRanges)) return scanTypeDeclaration(x) break } } function scanFunctionDeclaration(x: Parser.SyntaxNode) { const scope = new Scope(rootScope) for (const child of x.namedChildren) { switch (child.type) { case 'identifier': if (isVisible(child, visibleRanges)) { functions.push({start: child.startPosition, end: child.endPosition}); } break default: scanExpr(child, scope) } } } function scanVarDeclaration(x: Parser.SyntaxNode) { for (const varSpec of x.namedChildren) { for (const child of varSpec.namedChildren) { switch (child.type) { case 'identifier': if (isVisible(child, visibleRanges)) { variables.push({start: child.startPosition, end: child.endPosition}); } break default: scanExpr(child, rootScope) } } } } function scanTypeDeclaration(x: Parser.SyntaxNode) { for (const child of x.namedChildren) { scanExpr(child, rootScope) } } function scanExpr(x: Parser.SyntaxNode, scope: Scope) { switch (x.type) { case 'ERROR': return case 'func_literal': case 'method_spec': case 'block': case 'expression_case': case 'type_case': case 'for_statement': case 'if_statement': scope = new Scope(scope) break case 'parameter_declaration': case 'variadic_parameter_declaration': case 'var_spec': case 'const_spec': for (const id of x.namedChildren) { if (id.type == 'identifier') { scope.declareLocal(id.text) } } break case 'short_var_declaration': case 'range_clause': for (const id of x.firstChild!.namedChildren) { if (id.type == 'identifier') { scope.declareLocal(id.text) } } break case 'type_switch_statement': scope = new Scope(scope) if (x.firstNamedChild!.type == 'expression_list') { for (const id of x.firstNamedChild!.namedChildren) { scope.declareLocal(id.text) } } break case 'inc_statement': case 'dec_statement': scope.modifyLocal(x.firstChild!.text) break case 'assignment_statement': for (const id of x.firstChild!.namedChildren) { if (id.type == 'identifier') { scope.modifyLocal(id.text) } } break case 'call_expression': scanCall(x.firstChild!, scope) scanExpr(x.lastChild!, scope) return case 'identifier': scope.referenceLocal(x) if (isVisible(x, visibleRanges) && scope.isUnknown(x.text)) { variables.push({start: x.startPosition, end: x.endPosition}); } return case 'selector_expression': if (isVisible(x, visibleRanges) && scope.isPackage(x.firstChild!.text)) { variables.push({start: x.lastChild!.startPosition, end: x.lastChild!.endPosition}) } scanExpr(x.firstChild!, scope) scanExpr(x.lastChild!, scope) return case 'type_identifier': if (isVisible(x, visibleRanges)) { types.push({start: x.startPosition, end: x.endPosition}) } return } for (const child of x.namedChildren) { scanExpr(child, scope) } } function scanCall(x: Parser.SyntaxNode, scope: Scope) { switch (x.type) { case 'identifier': if (isVisible(x, visibleRanges) && scope.isUnknown(x.text)) { functions.push({start: x.startPosition, end: x.endPosition}) } scope.referenceLocal(x) return case 'selector_expression': if (isVisible(x, visibleRanges) && scope.isPackage(x.firstChild!.text)) { functions.push({start: x.lastChild!.startPosition, end: x.lastChild!.endPosition}) } scanExpr(x.firstChild!, scope) scanExpr(x.lastChild!, scope) return case 'unary_expression': scanCall(x.firstChild!, scope) return default: scanExpr(x, scope) } } scanSourceFile() for (const scope of allScopes) { for (const local of scope.modifiedLocals()) { underlines.push({start: local.startPosition, end: local.endPosition}) } } return new Map([ ['entity.name.function', functions], ['entity.name.type', types], ['variable', variables], ['markup.underline', underlines], ]) } export function colorTypescript(root: Parser.Tree, visibleRanges: {start: number, end: number}[]) { const functions: Range[] = [] const types: Range[] = [] const variables: Range[] = [] const keywords: Range[] = [] let visitedChildren = false let cursor = root.walk() let parents = [cursor.nodeType] while (true) { // Advance cursor if (visitedChildren) { if (cursor.gotoNextSibling()) { visitedChildren = false } else if (cursor.gotoParent()) { parents.pop() visitedChildren = true continue } else { break } } else { const parent = cursor.nodeType if (cursor.gotoFirstChild()) { parents.push(parent) visitedChildren = false } else { visitedChildren = true continue } } // Skip nodes that are not visible if (!visible(cursor, visibleRanges)) { visitedChildren = true continue } // Color tokens const parent = parents[parents.length - 1] switch (cursor.nodeType) { case 'identifier': if (parent == 'function') { functions.push({start: cursor.startPosition, end: cursor.endPosition}) } break case 'type_identifier': case 'predefined_type': types.push({start: cursor.startPosition, end: cursor.endPosition}) break case 'property_identifier': variables.push({start: cursor.startPosition, end: cursor.endPosition}) break case 'method_definition': const firstChild = cursor.currentNode().firstChild! switch (firstChild.text) { case 'get': case 'set': keywords.push({start: firstChild.startPosition, end: firstChild.endPosition}) } break case 'function_declaration': const functionName = cursor.currentNode().firstNamedChild! functions.push({start: functionName.startPosition, end: functionName.endPosition}) } } cursor.delete() return new Map([ ['entity.name.function', functions], ['entity.name.type', types], ['variable', variables], ['keyword', keywords], ]) } export function colorRuby(root: Parser.Tree, visibleRanges: {start: number, end: number}[]) { const controlKeywords = new Set(['while', 'until', 'if', 'unless', 'for', 'begin', 'elsif', 'else', 'ensure', 'when', 'case', 'do_block']) const classKeywords = new Set(['include', 'prepend', 'extend', 'private', 'protected', 'public', 'attr_reader', 'attr_writer', 'attr_accessor', 'attr', 'private_class_method', 'public_class_method']) const moduleKeywords = new Set(['module_function', ...classKeywords]) const functions: Range[] = [] const types: Range[] = [] const variables: Range[] = [] const keywords: Range[] = [] const controls: Range[] = [] const constants: Range[] = [] let visitedChildren = false let cursor = root.walk() let parents = [cursor.nodeType] function isChildOf(ancestor: string) { const parent = parents[parents.length - 1] const grandparent = parents[parents.length - 2] // class Foo; bar; end if (parent == ancestor) { return true } // class Foo; bar :thing; end if (parent == 'method_call' && grandparent == ancestor) { return true } return false } while (true) { // Advance cursor if (visitedChildren) { if (cursor.gotoNextSibling()) { visitedChildren = false } else if (cursor.gotoParent()) { parents.pop() visitedChildren = true continue } else { break } } else { const parent = cursor.nodeType if (cursor.gotoFirstChild()) { parents.push(parent) visitedChildren = false } else { visitedChildren = true continue } } // Skip nodes that are not visible if (!visible(cursor, visibleRanges)) { visitedChildren = true continue } // Color tokens const parent = parents[parents.length - 1] switch (cursor.nodeType) { case 'method': cursor.gotoFirstChild() cursor.gotoNextSibling() functions.push({start: cursor.startPosition, end: cursor.endPosition}) cursor.gotoParent() break case 'singleton_method': cursor.gotoFirstChild() cursor.gotoNextSibling() cursor.gotoNextSibling() cursor.gotoNextSibling() functions.push({start: cursor.startPosition, end: cursor.endPosition}) cursor.gotoParent() break case 'instance_variable': case 'class_variable': case 'global_variable': variables.push({start: cursor.startPosition, end: cursor.endPosition}) break case 'end': if (controlKeywords.has(parent)) { controls.push({start: cursor.startPosition, end: cursor.endPosition}) } else { keywords.push({start: cursor.startPosition, end: cursor.endPosition}) } break case 'constant': types.push({start: cursor.startPosition, end: cursor.endPosition}) break case 'symbol': constants.push({start: cursor.startPosition, end: cursor.endPosition}) break case 'method_call': { cursor.gotoFirstChild() const text = cursor.currentNode().text if (!moduleKeywords.has(text)) { functions.push({start: cursor.startPosition, end: cursor.endPosition}) } cursor.gotoParent() break } case 'call': cursor.gotoFirstChild() cursor.gotoNextSibling() cursor.gotoNextSibling() functions.push({start: cursor.startPosition, end: cursor.endPosition}) cursor.gotoParent() break case 'identifier': { const text = cursor.currentNode().text if (classKeywords.has(text) && isChildOf('class')) { keywords.push({start: cursor.startPosition, end: cursor.endPosition}) } else if (moduleKeywords.has(text) && isChildOf('module')) { keywords.push({start: cursor.startPosition, end: cursor.endPosition}) } break } } } cursor.delete() return new Map([ ['entity.name.function', functions], ['entity.name.type', types], ['variable', variables], ['keyword', keywords], ['keyword.control', controls], ['constant.language', constants], ]) } export function colorRust(root: Parser.Tree, visibleRanges: {start: number, end: number}[]) { const functions: Range[] = [] const types: Range[] = [] const variables: Range[] = [] const keywords: Range[] = [] let visitedChildren = false let cursor = root.walk() let parents = [cursor.nodeType] while (true) { // Advance cursor if (visitedChildren) { if (cursor.gotoNextSibling()) { visitedChildren = false } else if (cursor.gotoParent()) { parents.pop() visitedChildren = true continue } else { break } } else { const parent = cursor.nodeType if (cursor.gotoFirstChild()) { parents.push(parent) visitedChildren = false } else { visitedChildren = true continue } } // Skip nodes that are not visible if (!visible(cursor, visibleRanges)) { visitedChildren = true continue } // Color tokens const parent = parents[parents.length - 1] const grandparent = parents[parents.length - 2] switch (cursor.nodeType) { case 'identifier': if (parent == 'function_item' && grandparent == 'declaration_list') { variables.push({start: cursor.startPosition, end: cursor.endPosition}) } else if (parent == 'function_item') { functions.push({start: cursor.startPosition, end: cursor.endPosition}) } else if (parent == 'scoped_identifier' && grandparent == 'function_declarator') { functions.push({start: cursor.startPosition, end: cursor.endPosition}) } break case 'type_identifier': case 'primitive_type': types.push({start: cursor.startPosition, end: cursor.endPosition}) break case 'field_identifier': variables.push({start: cursor.startPosition, end: cursor.endPosition}) break case 'use_list': } } cursor.delete() return new Map([ ['entity.name.function', functions], ['entity.name.type', types], ['variable', variables], ['keyword', keywords], ]) } export function colorCpp(root: Parser.Tree, visibleRanges: {start: number, end: number}[]) { const functions: Range[] = [] const types: Range[] = [] const variables: Range[] = [] let visitedChildren = false let cursor = root.walk() let parents = [cursor.nodeType] while (true) { // Advance cursor if (visitedChildren) { if (cursor.gotoNextSibling()) { visitedChildren = false } else if (cursor.gotoParent()) { parents.pop() visitedChildren = true continue } else { break } } else { const parent = cursor.nodeType if (cursor.gotoFirstChild()) { parents.push(parent) visitedChildren = false } else { visitedChildren = true continue } } // Skip nodes that are not visible if (!visible(cursor, visibleRanges)) { visitedChildren = true continue } // Color tokens const parent = parents[parents.length - 1] const grandparent = parents[parents.length - 2] switch (cursor.nodeType) { case 'identifier': if (parent == 'function_declarator' || parent == 'scoped_identifier' && grandparent == 'function_declarator') { functions.push({start: cursor.startPosition, end: cursor.endPosition}) } break case 'type_identifier': types.push({start: cursor.startPosition, end: cursor.endPosition}) break case 'field_identifier': variables.push({start: cursor.startPosition, end: cursor.endPosition}) break } } cursor.delete() return new Map([ ['entity.name.function', functions], ['entity.name.type', types], ['variable', variables], ]) } function isVisible(x: Parser.SyntaxNode, visibleRanges: {start: number, end: number}[]) { for (const {start, end} of visibleRanges) { const overlap = x.startPosition.row <= end+1 && start-1 <= x.endPosition.row if (overlap) return true } return false } function visible(x: Parser.TreeCursor, visibleRanges: { start: number, end: number }[]) { for (const { start, end } of visibleRanges) { const overlap = x.startPosition.row <= end + 1 && start - 1 <= x.endPosition.row if (overlap) return true } return false } ================================================ FILE: src/extension.ts ================================================ import * as vscode from 'vscode' import * as Parser from 'web-tree-sitter' import * as path from 'path' import * as scopes from './scopes' import * as colors from './colors' // Be sure to declare the language in package.json and include a minimalist grammar. const languages: {[id: string]: {module: string, color: colors.ColorFunction, parser?: Parser}} = { 'go': {module: 'tree-sitter-go', color: colors.colorGo}, 'cpp': {module: 'tree-sitter-cpp', color: colors.colorCpp}, 'rust': {module: 'tree-sitter-rust', color: colors.colorRust}, 'ruby': {module: 'tree-sitter-ruby', color: colors.colorRuby}, 'typescript': {module: 'tree-sitter-typescript', color: colors.colorTypescript}, // TODO there is a separate JS grammar now 'javascript': {module: 'tree-sitter-javascript', color: colors.colorTypescript}, } // Create decoration types from scopes lazily const decorationCache = new Map() function decoration(scope: string): vscode.TextEditorDecorationType|undefined { // If we've already created a decoration for `scope`, use it if (decorationCache.has(scope)) { return decorationCache.get(scope) } // If `scope` is defined in the current theme, create a decoration for it const textmate = scopes.find(scope) if (textmate) { const decoration = createDecorationFromTextmate(textmate) decorationCache.set(scope, decoration) return decoration } // Otherwise, give up, there is no color available for this scope return undefined } function createDecorationFromTextmate(themeStyle: scopes.TextMateRuleSettings): vscode.TextEditorDecorationType { let options: vscode.DecorationRenderOptions = {} options.rangeBehavior = vscode.DecorationRangeBehavior.OpenOpen if (themeStyle.foreground) { options.color = themeStyle.foreground } if (themeStyle.background) { options.backgroundColor = themeStyle.background } if (themeStyle.fontStyle) { let parts: string[] = themeStyle.fontStyle.split(" ") parts.forEach((part) => { switch (part) { case "italic": options.fontStyle = "italic" break case "bold": options.fontWeight = "bold" break case "underline": options.textDecoration = "underline" break default: break } }) } return vscode.window.createTextEditorDecorationType(options) } // Load styles from the current active theme async function loadStyles() { await scopes.load() // Clear old styles for (const style of decorationCache.values()) { style.dispose() } decorationCache.clear() } // For some reason this crashes if we put it inside activate const initParser = Parser.init() // TODO this isn't a field, suppress package member coloring like Go // Called when the extension is first activated by user opening a file with the appropriate language export async function activate(context: vscode.ExtensionContext) { console.log("Activating tree-sitter...") // Parse of all visible documents const trees: {[uri: string]: Parser.Tree} = {} async function open(editor: vscode.TextEditor) { const language = languages[editor.document.languageId] if (language == null) return if (language.parser == null) { const absolute = path.join(context.extensionPath, 'parsers', language.module + '.wasm') const wasm = path.relative(process.cwd(), absolute) const lang = await Parser.Language.load(wasm) const parser = new Parser() parser.setLanguage(lang) language.parser = parser } const t = language.parser.parse(editor.document.getText()) // TODO don't use getText, use Parser.Input trees[editor.document.uri.toString()] = t colorUri(editor.document.uri) } // NOTE: if you make this an async function, it seems to cause edit anomalies function edit(edit: vscode.TextDocumentChangeEvent) { const language = languages[edit.document.languageId] if (language == null || language.parser == null) return updateTree(language.parser, edit) colorUri(edit.document.uri) } function updateTree(parser: Parser, edit: vscode.TextDocumentChangeEvent) { if (edit.contentChanges.length == 0) return const old = trees[edit.document.uri.toString()] for (const e of edit.contentChanges) { const startIndex = e.rangeOffset const oldEndIndex = e.rangeOffset + e.rangeLength const newEndIndex = e.rangeOffset + e.text.length const startPos = edit.document.positionAt(startIndex) const oldEndPos = edit.document.positionAt(oldEndIndex) const newEndPos = edit.document.positionAt(newEndIndex) const startPosition = asPoint(startPos) const oldEndPosition = asPoint(oldEndPos) const newEndPosition = asPoint(newEndPos) const delta = {startIndex, oldEndIndex, newEndIndex, startPosition, oldEndPosition, newEndPosition} old.edit(delta) } const t = parser.parse(edit.document.getText(), old) // TODO don't use getText, use Parser.Input trees[edit.document.uri.toString()] = t } function asPoint(pos: vscode.Position): Parser.Point { return {row: pos.line, column: pos.character} } function close(doc: vscode.TextDocument) { delete trees[doc.uri.toString()] } function colorUri(uri: vscode.Uri) { for (const editor of vscode.window.visibleTextEditors) { if (editor.document.uri == uri) { colorEditor(editor) } } } const warnedScopes = new Set() function colorEditor(editor: vscode.TextEditor) { const t = trees[editor.document.uri.toString()] if (t == null) return const language = languages[editor.document.languageId] if (language == null) return const scopes = language.color(t, visibleLines(editor)) for (const scope of scopes.keys()) { const dec = decoration(scope) if (dec) { const ranges = scopes.get(scope)!.map(range) editor.setDecorations(dec, ranges) } else if (!warnedScopes.has(scope)) { console.warn(scope, 'was not found in the current theme') warnedScopes.add(scope) } } for (const scope of decorationCache.keys()) { if (!scopes.has(scope)) { const dec = decorationCache.get(scope)! editor.setDecorations(dec, []) } } } async function colorAllOpen() { for (const editor of vscode.window.visibleTextEditors) { await open(editor) } } // Load active color theme async function onChangeConfiguration(event: vscode.ConfigurationChangeEvent) { let colorizationNeedsReload: boolean = event.affectsConfiguration("workbench.colorTheme") || event.affectsConfiguration("editor.tokenColorCustomizations") if (colorizationNeedsReload) { await loadStyles() colorAllOpen() } } context.subscriptions.push(vscode.workspace.onDidChangeConfiguration(onChangeConfiguration)) context.subscriptions.push(vscode.window.onDidChangeVisibleTextEditors(colorAllOpen)) context.subscriptions.push(vscode.workspace.onDidChangeTextDocument(edit)) context.subscriptions.push(vscode.workspace.onDidCloseTextDocument(close)) context.subscriptions.push(vscode.window.onDidChangeTextEditorVisibleRanges(change => colorEditor(change.textEditor))) // Don't wait for the initial color, it takes too long to inspect the themes and causes VSCode extension host to hang async function activateLazily() { await loadStyles() await initParser colorAllOpen() } activateLazily() } function visibleLines(editor: vscode.TextEditor) { return editor.visibleRanges.map(range => { const start = range.start.line const end = range.end.line return {start, end} }) } function range(x: colors.Range): vscode.Range { return new vscode.Range(x.start.row, x.start.column, x.end.row, x.end.column) } // this method is called when your extension is deactivated export function deactivate() {} ================================================ FILE: src/print.ts ================================================ // import extension = require('./extension') import Parser = require('web-tree-sitter') import fs = require('fs') testRust() async function testRust() { await Parser.init() const parser = new Parser() const wasm = 'parsers/tree-sitter-rust.wasm' const lang = await Parser.Language.load(wasm) parser.setLanguage(lang) const text = fs.readFileSync('examples/rust/scratch.rs', {encoding: 'utf-8'}) const tree = parser.parse(text) const lines = text.split('\n') const maxLine = maxWidth(lines) for (let line = 0; line < lines.length; line++) { const types: string[] = [] collectTypes(tree.rootNode, line, types) let acc = lines[line] for (let i = acc.length; i < maxLine + 1; i++) { acc = acc + ' ' } for (const t of types) { acc = acc + ' ' + t } console.log(acc) } } function maxWidth(lines: string[]): number { let max = 0 for (const line of lines) { if (line.length > max) max = line.length } return max } function collectTypes(node: Parser.SyntaxNode, line: number, types: string[]) { if (node.startPosition.row == line) { if (node.endPosition.row == line) { types.push(node.toString()) } else { types.push(node.type) for (const child of node.children) { collectTypes(child, line, types) } } } else { for (const child of node.children) { collectTypes(child, line, types) } } } ================================================ FILE: src/scopes.ts ================================================ import * as vscode from 'vscode' import * as path from 'path' import * as fs from 'fs' import * as jsonc from "jsonc-parser" export interface TextMateRule { scope: string|string[] settings: TextMateRuleSettings } export interface TextMateRuleSettings { foreground: string | undefined background: string | undefined fontStyle: string | undefined } // Current theme colors const colors = new Map() export function find(scope: string): TextMateRuleSettings|undefined { return colors.get(scope) } // Load all textmate scopes in the currently active theme export async function load() { // Remove any previous theme colors.clear() // Find out current color theme const themeName = vscode.workspace.getConfiguration("workbench").get("colorTheme") if (typeof themeName != 'string') { console.warn('workbench.colorTheme is', themeName) return } // Try to load colors from that theme try { await loadThemeNamed(themeName) } catch(e) { console.warn('failed to load theme', themeName, e) } } // Find current theme on disk async function loadThemeNamed(themeName: string) { for (const extension of vscode.extensions.all) { const extensionPath: string = extension.extensionPath const extensionPackageJsonPath: string = path.join(extensionPath, "package.json") if (!await checkFileExists(extensionPackageJsonPath)) { continue } const packageJsonText: string = await readFileText(extensionPackageJsonPath) const packageJson: any = jsonc.parse(packageJsonText) if (packageJson.contributes && packageJson.contributes.themes) { for (const theme of packageJson.contributes.themes) { const id = theme.id || theme.label if (id == themeName) { const themeRelativePath: string = theme.path const themeFullPath: string = path.join(extensionPath, themeRelativePath) await loadThemeFile(themeFullPath) } } } } } async function loadThemeFile(themePath: string) { if (await checkFileExists(themePath)) { const themeContentText: string = await readFileText(themePath) const themeContent: any = jsonc.parse(themeContentText) if (themeContent && themeContent.tokenColors) { loadColors(themeContent.tokenColors) if (themeContent.include) { // parse included theme file const includedThemePath: string = path.join(path.dirname(themePath), themeContent.include) await loadThemeFile(includedThemePath) } } } } function loadColors(textMateRules: TextMateRule[]): void { for (const rule of textMateRules) { if (typeof rule.scope == 'string') { if (!colors.has(rule.scope)) { colors.set(rule.scope, rule.settings) } } else if (rule.scope instanceof Array) { for (const scope of rule.scope) { if (!colors.has(scope)) { colors.set(scope, rule.settings) } } } } } function checkFileExists(filePath: string): Promise { return new Promise((resolve, reject) => { fs.stat(filePath, (err, stats) => { if (stats && stats.isFile()) { resolve(true) } else { console.warn('no such file', filePath) resolve(false) } }) }) } function readFileText(filePath: string, encoding: string = "utf8"): Promise { return new Promise((resolve, reject) => { fs.readFile(filePath, encoding, (err, data) => { if (err) { reject(err) } else { resolve(data) } }) }) } ================================================ FILE: src/test.ts ================================================ import Parser = require('web-tree-sitter') import colors = require('./colors') type Assert = [string, string|{not:string}] type TestCase = [string, ...Assert[]] const goTests: TestCase[] = [ [ `package p; func f() int { }`, ['f', 'entity.name.function'], ['int', 'entity.name.type'] ], [ `package p; type Foo struct { x int }`, ['Foo', 'entity.name.type'], ['x', {not: 'variable'}] ], [ `package p; type Foo interface { GetX() int }`, ['Foo', 'entity.name.type'], ['int', 'entity.name.type'], ['GetX', {not: 'variable'}] ], [ `package p; func f() { x := 1; x := 2 }`, ['x', 'markup.underline'] ], [ `package p; func f(foo T) { foo.Foo() }`, ['Foo', {not: 'entity.name.function'}] ], [ `package p; func f() { Foo() }`, ['Foo', 'entity.name.function'] ], [ `package p; import "foo"; func f() { foo.Foo() }`, ['Foo', 'entity.name.function'] ], [ `package p; import "foo"; func f(foo T) { foo.Foo() }`, ['Foo', {not: 'entity.name.function'}] ], [ `package p; func f(x other.T) { }`, ['T', 'entity.name.type'], ], [ `package p; var _ = f(Foo{})`, ['Foo', 'entity.name.type'], ], [ `package p; import (foo "foobar"); var _ = foo.Bar()`, ['foo', {not:'variable'}], ['Bar', 'entity.name.function'], ], [ `package p func f(a int) int { switch a { case 1: x := 1 return x case 2: x := 2 return x } }`, ['x', {not:'markup.underline'}] ], [ `package p func f(a interface{}) int { switch a.(type) { case *int: x := 1 return x case *int: x := 2 return x } }`, ['x', {not:'markup.underline'}] ], [ `package p func f(a interface{}) int { for i := range 10 { print(i) } for i := range 10 { print(i) } }`, ['i', {not:'markup.underline'}] ], [ `package p func f(a interface{}) int { if i := 1; i < 10 { print(i) } if i := 1; i < 10 { print(i) } }`, ['i', {not:'markup.underline'}] ], [ `package p func f(a interface{}) { if aa, ok := a.(*type); ok { print(aa) } }`, ['aa', {not:'variable'}] ], [ `package p func f(a interface{}) { switch aa := a.(type) { case *int: print(aa) } }`, ['aa', {not:'variable'}] ], [ `package p func f() { switch aa.(type) { case *int: print(aa) } }`, ['aa', 'variable'] ], [ `package p func f(a interface{}) { switch aa := a.(type) { case *int: print(aa) } switch aa := a.(type) { case *int: print(aa) } }`, ['aa', {not:'markup.underline'}] ], [ `package p func f(a ...int) { print(a) }`, ['a', {not:'variable'}] ], [ `package p type Foo interface { foo(i int) }`, ['i', {not: 'variable'}] ], [ `package p type Foo interface { foo(i int) bar(i int) }`, ['i', {not: 'markup.underline'}] ], ] test(goTests, 'parsers/tree-sitter-go.wasm', colors.colorGo) const rubyTests: TestCase[] = [ [ `def x.f 1 end`, ['f', 'entity.name.function'], ], [ `def f 1 end`, ['f', 'entity.name.function'], ], [ `class C def f @x = 1 end end`, ['@x', 'variable'], ], [ `class C private def f 1 end end`, ['C', 'entity.name.type'], ['private', 'keyword'], ['f', 'entity.name.function'], ['end', 'keyword'], ], [ `class C private :f def f 1 end end`, ['C', 'entity.name.type'], ['private', 'keyword'], [':f', 'constant.language'], ['private', {not:'entity.name.function'}], ['f', 'entity.name.function'], ['end', 'keyword'], ], [ `module M private def f 1 end end`, ['M', 'entity.name.type'], ['private', 'keyword'], ['f', 'entity.name.function'], ['end', 'keyword'], ], [ `module M private :f def f 1 end end`, ['M', 'entity.name.type'], ['private', 'keyword'], ['private', {not:'entity.name.function'}], [':f', 'constant.language'], ['f', 'entity.name.function'], ['end', 'keyword'], ], [ `while true puts "Hi" end`, ['end', 'keyword.control'], ['end', {not: 'keyword'}], ], [ `foo 1`, ['foo', 'entity.name.function'], ], [ `foo.bar`, ['bar', 'entity.name.function'], ], ] test(rubyTests, 'parsers/tree-sitter-ruby.wasm', colors.colorRuby) async function test(testCases: TestCase[], wasm: string, color: colors.ColorFunction) { await Parser.init() const parser = new Parser() const lang = await Parser.Language.load(wasm) parser.setLanguage(lang) for (const [src, ...expect] of testCases) { const tree = parser.parse(src) const scope2ranges = color(tree, [{start: 0, end: tree.rootNode.endPosition.row}]) const code2scopes = new Map>() for (const [scope, ranges] of scope2ranges) { for (const range of ranges) { const start = index(src, range.start) const end = index(src, range.end) const code = src.substring(start, end) if (!code2scopes.has(code)) { code2scopes.set(code, new Set()) } code2scopes.get(code)!.add(scope) } } function printSrcAndTree() { console.error('Source:\t' + src) console.error('Parsed:\t' + tree.rootNode.toString()) } for (const [code, assert] of expect) { if (typeof assert == 'string') { const scope = assert if (!code2scopes.has(code)) { console.error(`Error:\tcode (${code}) was not found in (${join(code2scopes.keys())})`) printSrcAndTree() continue } const foundScopes = code2scopes.get(code)! if (!foundScopes.has(scope)) { console.error(`Error:\tscope (${scope}) was not among the scopes for (${code}) (${join(foundScopes.keys())})`) printSrcAndTree() continue } } else { const scope = assert.not if (!code2scopes.has(code)) { continue } const foundScopes = code2scopes.get(code)! if (foundScopes.has(scope)) { console.error(`Error:\tbanned scope (${scope}) was among the scopes for (${code}) (${join(foundScopes.keys())})`) printSrcAndTree() continue } } } } } function index(code: string, point: Parser.Point): number { let row = 0 let column = 0 for (let i = 0; i < code.length; i++) { if (row == point.row && column == point.column) { return i } if (code[i] == '\n') { row++ column = 0 } else { column++ } } return code.length } function join(strings: IterableIterator) { var result = '' for (const s of strings) { result = result + s + ', ' } return result.substring(0, result.length - 2) } ================================================ FILE: textmate/cpp.tmLanguage.json ================================================ { "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", "version": "https://github.com/atom/language-c/commit/3a269f88b12e512fb9495dc006a1dabf325d3d7f", "name": "C++", "scopeName": "source.cpp", "patterns": [ { "include": "#keywords" }, { "include": "#constants" }, { "include": "#strings" }, { "include": "#comments" }, { "include": "#numbers" }, { "include": "#preprocessor-rule-enabled" }, { "include": "#preprocessor-rule-disabled" }, { "include": "#preprocessor-rule-conditional" }, { "begin": "(?x)\n^\\s* ((\\#)\\s*define) \\s+ # define\n((?[a-zA-Z_$][\\w$]*)) # macro name\n(?:\n (\\()\n (\n \\s* \\g \\s* # first argument\n ((,) \\s* \\g \\s*)* # additional arguments\n (?:\\.\\.\\.)? # varargs ellipsis?\n )\n (\\))\n)?", "beginCaptures": { "1": { "name": "keyword.other.directive.define.c" }, "2": { "name": "punctuation.definition.directive.c" }, "5": { "name": "punctuation.definition.parameters.begin.c" }, "8": { "name": "punctuation.separator.parameters.c" }, "9": { "name": "punctuation.definition.parameters.end.c" } }, "end": "(?=(?://|/\\*))|(?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.c" } }, "name": "string.quoted.other.lt-gt.include.c" } ] } ], "repository": { "keywords": { "patterns": [ { "match": "\\b(alignas|alignof|and|and_eq|asm|atomic_cancel|atomic_commit|atomic_noexcept|auto|bitand|bitor|bool|char|char8_t|char16_t|char32_t|class|compl|concept|const|consteval|constexpr|const_cast|decltype|double|dynamic_cast|enum|explicit|export|extern|float|friend|inline|int|long|mutable|namespace|noexcept|not|not_eq|nullptr|operator|or|or_eq|private|protected|public|reflexpr|register|reinterpret_cast|requires|short|signed|sizeof|static|static_assert|static_cast|struct|template|this|thread_local|typedef|typeid|typename|union|unsigned|using|virtual|void|volatile|wchar_t|xor|xor_eq)\\b", "captures": { "1": { "name": "keyword.other.cpp" } } }, { "match": "\\b(override|final|audit|axiom|import|module|transaction_safe|transaction_safe_dynamic)\\b", "captures": { "1": { "name": "keyword.other.cpp" } } }, { "match": "\\b(_Pragma)\\b", "captures": { "1": { "name": "keyword.other.cpp" } } }, { "match": "\\b(break|case|catch|continue|co_await|co_return|co_yield|default|delete|do|else|for|goto|if|new|return|switch|synchronized|throw|try|while)\\b", "captures": { "1": { "name": "keyword.control.cpp" } } }, { "comment": "Terminator", "match": ";", "name": "keyword.other.semi.go" } ] }, "constants": { "patterns": [ { "match": "\\b(NULL|true|false|TRUE|FALSE)\\b", "name": "constant.numeric.cpp" } ] }, "strings": { "patterns": [ { "begin": "(u|u8|U|L)?R\"(?:([^ ()\\\\\\t]{0,16})|([^ ()\\\\\\t]*))\\(", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.cpp" }, "1": { "name": "meta.encoding.cpp" }, "3": { "name": "invalid.illegal.delimiter-too-long.cpp" } }, "end": "\\)\\2(\\3)\"", "endCaptures": { "0": { "name": "punctuation.definition.string.end.cpp" }, "1": { "name": "invalid.illegal.delimiter-too-long.cpp" } }, "name": "string.quoted.double.raw.cpp" }, { "begin": "(u|u8|U|L)?\"", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.c" }, "1": { "name": "meta.encoding.cpp" } }, "end": "\"", "endCaptures": { "0": { "name": "punctuation.definition.string.end.c" } }, "name": "string.quoted.double.c", "patterns": [ { "include": "#string_escaped_char" }, { "include": "#string_placeholder" }, { "include": "#line_continuation_character" } ] }, { "begin": "'", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.c" } }, "end": "'", "endCaptures": { "0": { "name": "punctuation.definition.string.end.c" } }, "name": "string.quoted.single.c", "patterns": [ { "include": "#string_escaped_char" }, { "include": "#line_continuation_character" } ] } ] }, "string_escaped_char": { "patterns": [ { "match": "(?x)\\\\ (\n\\\\ |\n[abefnprtv'\"?] |\n[0-3]\\d{,2} |\n[4-7]\\d? |\nx[a-fA-F0-9]{,2} |\nu[a-fA-F0-9]{,4} |\nU[a-fA-F0-9]{,8} )", "name": "constant.character.escape.c" }, { "match": "\\\\.", "name": "invalid.illegal.unknown-escape.c" } ] }, "string_placeholder": { "patterns": [ { "match": "(?x) %\n(\\d+\\$)? # field (argument #)\n[#0\\- +']* # flags\n[,;:_]? # separator character (AltiVec)\n((-?\\d+)|\\*(-?\\d+\\$)?)? # minimum field width\n(\\.((-?\\d+)|\\*(-?\\d+\\$)?)?)? # precision\n(hh|h|ll|l|j|t|z|q|L|vh|vl|v|hv|hl)? # length modifier\n[diouxXDOUeEfFgGaACcSspn%] # conversion type", "name": "constant.other.placeholder.c" }, { "match": "(%)(?!\"\\s*(PRI|SCN))", "captures": { "1": { "name": "invalid.illegal.placeholder.c" } } } ] }, "comments": { "patterns": [ { "captures": { "1": { "name": "meta.toc-list.banner.block.c" } }, "match": "^/\\* =(\\s*.*?)\\s*= \\*/$\\n?", "name": "comment.block.c" }, { "begin": "/\\*", "beginCaptures": { "0": { "name": "punctuation.definition.comment.begin.c" } }, "end": "\\*/", "endCaptures": { "0": { "name": "punctuation.definition.comment.end.c" } }, "name": "comment.block.c" }, { "match": "\\*/.*\\n", "name": "invalid.illegal.stray-comment-end.c" }, { "captures": { "1": { "name": "meta.toc-list.banner.line.c" } }, "match": "^// =(\\s*.*?)\\s*=\\s*$\\n?", "name": "comment.line.banner.cpp" }, { "begin": "(^[ \\t]+)?(?=//)", "beginCaptures": { "1": { "name": "punctuation.whitespace.comment.leading.cpp" } }, "end": "(?!\\G)", "patterns": [ { "begin": "//", "beginCaptures": { "0": { "name": "punctuation.definition.comment.cpp" } }, "end": "(?=\\n)", "name": "comment.line.double-slash.cpp", "patterns": [ { "include": "#line_continuation_character" } ] } ] } ] }, "numbers": { "patterns": [ { "match": "\\b((0(x|X)[0-9a-fA-F]([0-9a-fA-F']*[0-9a-fA-F])?)|(0(b|B)[01]([01']*[01])?)|(([0-9]([0-9']*[0-9])?\\.?[0-9]*([0-9']*[0-9])?)|(\\.[0-9]([0-9']*[0-9])?))((e|E)(\\+|-)?[0-9]([0-9']*[0-9])?)?)(L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\\b", "name": "constant.numeric.c" } ] }, "line_continuation_character": { "patterns": [ { "match": "(\\\\)\\n", "captures": { "1": { "name": "constant.character.escape.line-continuation.c" } } } ] }, "preprocessor-rule-conditional": { "patterns": [ { "begin": "^\\s*((#)\\s*if(?:n?def)?\\b)", "beginCaptures": { "0": { "name": "meta.preprocessor.c" }, "1": { "name": "keyword.other.directive.conditional.c" }, "2": { "name": "punctuation.definition.directive.c" } }, "end": "^\\s*((#)\\s*endif\\b)", "endCaptures": { "0": { "name": "meta.preprocessor.c" }, "1": { "name": "keyword.other.directive.conditional.c" }, "2": { "name": "punctuation.definition.directive.c" } }, "patterns": [ { "begin": "\\G(?=.)(?!//|/\\*(?!.*\\\\\\s*\\n))", "end": "(?=//)|(?=/\\*(?!.*\\\\\\s*\\n))|(?=+!]+|\\(\\)|\\[\\]))\n)\n\\s*(\\()", "beginCaptures": { "2": { "name": "punctuation.section.arguments.begin.bracket.round.c" } }, "end": "(\\))|(?=|<(?!<)|>(?!>))", "name": "keyword.operator.comparison.go" }, { "match": "(&&|\\|\\||!)", "name": "keyword.operator.logical.go" }, { "match": "(=|\\+=|\\-=|\\|=|\\^=|\\*=|/=|:=|%=|<<=|>>=|&\\^=|&=)", "name": "keyword.operator.assignment.go" }, { "match": "(\\+|\\-|\\*|/|%)", "name": "keyword.operator.arithmetic.go" }, { "match": "(&(?!\\^)|\\||\\^|&\\^|<<|>>)", "name": "keyword.operator.arithmetic.bitwise.go" }, { "match": "\\.\\.\\.", "name": "keyword.operator.ellipsis.go" } ] }, "runes": { "patterns": [ { "begin": "'", "end": "'", "name": "string.quoted.rune.go", "patterns": [ { "match": "\\G(\\\\([0-7]{3}|[abfnrtv\\\\'\"]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})|.)(?=')", "name": "constant.other.rune.go" }, { "match": "[^']+", "name": "invalid.illegal.unknown-rune.go" } ] } ] }, "string_escaped_char": { "patterns": [ { "match": "\\\\([0-7]{3}|[abfnrtv\\\\'\"]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})", "name": "constant.character.escape.go" }, { "match": "\\\\[^0-7xuUabfnrtv\\'\"]", "name": "invalid.illegal.unknown-escape.go" } ] }, "string_placeholder": { "patterns": [ { "match": "%(\\[\\d+\\])?([\\+#\\-0\\x20]{,2}((\\d+|\\*)?(\\.?(\\d+|\\*|(\\[\\d+\\])\\*?)?(\\[\\d+\\])?)?))?[vT%tbcdoqxXUbeEfFgGsp]", "name": "constant.other.placeholder.go" } ] } } } ================================================ FILE: textmate/ruby.tmLanguage.json ================================================ { "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", "name": "Ruby", "scopeName": "source.ruby", "patterns": [ { "match": "\\b(__ENCODING__|__LINE__|__FILE__|alias|and|class|def|module|not|or|self|super|undef)\\b", "name": "keyword.other.ruby" }, { "match": "\\b(BEGIN|END|begin|break|case|do|else|elsif|ensure|for|if|in|next|redo|rescue|retry|return|then|unless|until|when|while|yield)\\b", "name": "keyword.control.ruby" }, { "match": "\\bdefined[?]", "name": "keyword.control.ruby" }, { "comment": "These aren't actual keywords, but they're builtin methods that basically function as keywords", "match": "\\b(new|loop|raise|fail|catch|throw)\\b", "name": "keyword.control.special-method.ruby" }, { "comment": "These aren't actual keywords, but they're builtin methods that basically function as keywords", "match": "\\b(refine|using)\\b(?![?!])", "name": "keyword.other.special-method.ruby" }, { "match": "\\b(false|nil|true)\\b", "name": "constant.numeric.language.ruby" }, { "match": "\\b\\d(?>_?\\d)*(?=\\.\\d|[eE])(\\.\\d(?>_?\\d)*)?([eE][-+]?\\d(?>_?\\d)*)?r?i?\\b", "name": "constant.numeric.float.ruby" }, { "match": "\\b(0|(0[dD]\\d|[1-9])(?>_?\\d)*)r?i?\\b", "name": "constant.numeric.integer.ruby" }, { "match": "\\b0[xX]\\h(?>_?\\h)*r?i?\\b", "name": "constant.numeric.hex.ruby" }, { "match": "\\b0[bB][01](?>_?[01])*r?i?\\b", "name": "constant.numeric.binary.ruby" }, { "match": "\\b0([oO]?[0-7](?>_?[0-7])*)?r?i?\\b", "name": "constant.numeric.octal.ruby" }, { "comment": "Needs higher precidence than regular expressions.", "match": "(?~(?:\\[,|&;]\n\t\t\t | [\\s;]if\\s\t\t\t# keywords\n\t\t\t | [\\s;]elsif\\s\n\t\t\t | [\\s;]while\\s\n\t\t\t | [\\s;]unless\\s\n\t\t\t | [\\s;]when\\s\n\t\t\t | [\\s;]assert_match\\s\n\t\t\t | [\\s;]or\\s\t\t\t# boolean opperators\n\t\t\t | [\\s;]and\\s\n\t\t\t | [\\s;]not\\s\n\t\t\t | [\\s.]index\\s\t\t\t# methods\n\t\t\t | [\\s.]scan\\s\n\t\t\t | [\\s.]sub\\s\n\t\t\t | [\\s.]sub!\\s\n\t\t\t | [\\s.]gsub\\s\n\t\t\t | [\\s.]gsub!\\s\n\t\t\t | [\\s.]match\\s\n\t\t\t )\n\t\t\t | (?<= # or a look-behind with line anchor:\n\t\t\t ^when\\s # duplication necessary due to limits of regex\n\t\t\t | ^if\\s\n\t\t\t | ^elsif\\s\n\t\t\t | ^while\\s\n\t\t\t | ^unless\\s\n\t\t\t )\n\t\t\t )\n\t\t\t \\s*((/))(?![*+{}?])\n\t\t\t", "captures": { "1": { "name": "string.regexp.classic.ruby" } }, "comment": "regular expressions (normal)\n\t\t\twe only start a regexp if the character before it (excluding whitespace)\n\t\t\tis what we think is before a regexp\n\t\t\t", "contentName": "string.regexp.classic.ruby", "end": "((/[eimnosux]*))", "patterns": [ { "include": "#regex_sub" } ] }, { "begin": "^=begin", "comment": "multiline comments", "end": "^=end", "name": "comment.block.documentation.ruby" }, { "begin": "(^[ \\t]+)?(?=#)", "beginCaptures": { "1": { "name": "punctuation.whitespace.comment.leading.ruby" } }, "end": "(?!\\G)", "patterns": [ { "begin": "#", "end": "\\n", "name": "comment.line.number-sign.ruby" } ] }, { "comment": "\n\t\t\tmatches questionmark-letters.\n\n\t\t\texamples (1st alternation = hex):\n\t\t\t?\\x1 ?\\x61\n\n\t\t\texamples (2nd alternation = octal):\n\t\t\t?\\0 ?\\07 ?\\017\n\n\t\t\texamples (3rd alternation = escaped):\n\t\t\t?\\n ?\\b\n\n\t\t\texamples (4th alternation = meta-ctrl):\n\t\t\t?\\C-a ?\\M-a ?\\C-\\M-\\C-\\M-a\n\n\t\t\texamples (4th alternation = normal):\n\t\t\t?a ?A ?0 \n\t\t\t?* ?\" ?( \n\t\t\t?. ?#\n\t\t\t\n\t\t\t\n\t\t\tthe negative lookbehind prevents against matching\n\t\t\tp(42.tainted?)\n\t\t\t", "match": "(?<<[-~](\"?)((?:[_\\w]+_|)HTML)\\b\\1))", "comment": "Heredoc with embedded html", "end": "(?!\\G)", "name": "meta.embedded.block.html", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)HTML)\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "text.html", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "text.html.basic" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)XML)\\b\\1))", "comment": "Heredoc with embedded xml", "end": "(?!\\G)", "name": "meta.embedded.block.xml", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)XML)\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "text.xml", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "text.xml" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)SQL)\\b\\1))", "comment": "Heredoc with embedded sql", "end": "(?!\\G)", "name": "meta.embedded.block.sql", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)SQL)\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "source.sql", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "source.sql" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)CSS)\\b\\1))", "comment": "Heredoc with embedded css", "end": "(?!\\G)", "name": "meta.embedded.block.css", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)CSS)\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "source.css", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "source.css" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)CPP)\\b\\1))", "comment": "Heredoc with embedded c++", "end": "(?!\\G)", "name": "meta.embedded.block.c++", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)CPP)\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "source.c++", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "source.c++" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)C)\\b\\1))", "comment": "Heredoc with embedded c", "end": "(?!\\G)", "name": "meta.embedded.block.c", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)C)\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "source.c", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "source.c" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)(?:JS|JAVASCRIPT))\\b\\1))", "comment": "Heredoc with embedded javascript", "end": "(?!\\G)", "name": "meta.embedded.block.js", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)(?:JS|JAVASCRIPT))\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "source.js", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "source.js" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)JQUERY)\\b\\1))", "comment": "Heredoc with embedded jQuery javascript", "end": "(?!\\G)", "name": "meta.embedded.block.js.jquery", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)JQUERY)\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "source.js.jquery", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "source.js.jquery" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)(?:SH|SHELL))\\b\\1))", "comment": "Heredoc with embedded shell", "end": "(?!\\G)", "name": "meta.embedded.block.shell", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)(?:SH|SHELL))\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "source.shell", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "source.shell" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)LUA)\\b\\1))", "comment": "Heredoc with embedded lua", "end": "(?!\\G)", "name": "meta.embedded.block.lua", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)LUA)\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "source.lua", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "source.lua" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?=(?><<[-~](\"?)((?:[_\\w]+_|)RUBY)\\b\\1))", "comment": "Heredoc with embedded ruby", "end": "(?!\\G)", "name": "meta.embedded.block.ruby", "patterns": [ { "begin": "(?><<[-~](\"?)((?:[_\\w]+_|)RUBY)\\b\\1)", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "contentName": "source.ruby", "end": "\\s*\\2$\\n?", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "source.ruby" }, { "include": "#escaped_char" } ] } ] }, { "begin": "(?>=\\s*<<(\\w+))", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "end": "^\\1$", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "#escaped_char" } ] }, { "begin": "(?><<[-~](\\w+))", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "comment": "heredoc with indented terminator", "end": "\\s*\\1$", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.unquoted.heredoc.ruby", "patterns": [ { "include": "#heredoc" }, { "include": "#interpolated_ruby" }, { "include": "#escaped_char" } ] }, { "begin": "(?<=\\{|do|\\{\\s|do\\s)(\\|)", "captures": { "1": { "name": "punctuation.separator.arguments.ruby" } }, "end": "(?", "name": "punctuation.separator.key-value" }, { "match": "->", "name": "support.function.kernel.lambda.ruby" }, { "match": "<<=|%=|&{1,2}=|\\*=|\\*\\*=|\\+=|-=|\\^=|\\|{1,2}=|<<", "name": "keyword.operator.assignment.augmented.ruby" }, { "match": "<=>|<(?!<|=)|>(?!<|=|>)|<=|>=|===|==|=~|!=|!~|(?<=[ \\t])\\?", "name": "keyword.operator.comparison.ruby" }, { "match": "(?>", "name": "keyword.operator.other.ruby" }, { "match": ";", "name": "punctuation.separator.statement.ruby" }, { "match": ",", "name": "punctuation.separator.object.ruby" }, { "captures": { "1": { "name": "punctuation.separator.namespace.ruby" } }, "comment": "Mark as namespace separator if double colons followed by capital letter", "match": "(::)\\s*(?=[A-Z])" }, { "captures": { "1": { "name": "punctuation.separator.method.ruby" } }, "comment": "Mark as method separator if double colons not followed by capital letter", "match": "(\\.|::)\\s*(?![A-Z])" }, { "comment": "Must come after method and constant separators to prefer double colons", "match": ":", "name": "punctuation.separator.other.ruby" }, { "match": "\\{", "name": "punctuation.section.scope.begin.ruby" }, { "match": "\\}", "name": "punctuation.section.scope.end.ruby" }, { "match": "\\[", "name": "punctuation.section.array.begin.ruby" }, { "match": "\\]", "name": "punctuation.section.array.end.ruby" }, { "match": "\\(|\\)", "name": "punctuation.section.function.ruby" } ], "repository": { "escaped_char": { "match": "\\\\(?:[0-7]{1,3}|x[\\da-fA-F]{1,2}|.)", "name": "constant.character.escape.ruby" }, "heredoc": { "begin": "^<<[-~]?\\w+", "end": "$", "patterns": [ { "include": "$self" } ] }, "interpolated_ruby": { "patterns": [ { "begin": "#\\{", "beginCaptures": { "0": { "name": "punctuation.section.embedded.begin.ruby" } }, "contentName": "source.ruby", "end": "(\\})", "endCaptures": { "0": { "name": "punctuation.section.embedded.end.ruby" }, "1": { "name": "source.ruby" } }, "name": "meta.embedded.line.ruby", "patterns": [ { "include": "#nest_curly_and_self" }, { "include": "$self" } ], "repository": { "nest_curly_and_self": { "patterns": [ { "begin": "\\{", "captures": { "0": { "name": "punctuation.section.scope.ruby" } }, "end": "\\}", "patterns": [ { "include": "#nest_curly_and_self" } ] }, { "include": "$self" } ] } } }, { "captures": { "1": { "name": "punctuation.definition.variable.ruby" } }, "match": "(#@)[a-zA-Z_]\\w*", "name": "variable.other.readwrite.instance.ruby" }, { "captures": { "1": { "name": "punctuation.definition.variable.ruby" } }, "match": "(#@@)[a-zA-Z_]\\w*", "name": "variable.other.readwrite.class.ruby" }, { "captures": { "1": { "name": "punctuation.definition.variable.ruby" } }, "match": "(#\\$)[a-zA-Z_]\\w*", "name": "variable.other.readwrite.global.ruby" } ] }, "percent_literals": { "patterns": [ { "begin": "%i(?:([(\\[{<])|([^\\w\\s]|_))", "beginCaptures": { "0": { "name": "punctuation.section.array.begin.ruby" } }, "end": "[)\\]}>]\\2|\\1\\2", "endCaptures": { "0": { "name": "punctuation.section.array.end.ruby" } }, "name": "meta.array.symbol.ruby", "patterns": [ { "begin": "\\G(?<=\\()(?!\\))", "end": "(?=\\))", "patterns": [ { "include": "#parens" }, { "include": "#symbol" } ] }, { "begin": "\\G(?<=\\[)(?!\\])", "end": "(?=\\])", "patterns": [ { "include": "#brackets" }, { "include": "#symbol" } ] }, { "begin": "\\G(?<=\\{)(?!\\})", "end": "(?=\\})", "patterns": [ { "include": "#braces" }, { "include": "#symbol" } ] }, { "begin": "\\G(?<=<)(?!>)", "end": "(?=>)", "patterns": [ { "include": "#angles" }, { "include": "#symbol" } ] }, { "include": "#symbol" } ], "repository": { "angles": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\<|\\\\>", "name": "constant.other.symbol.ruby" }, { "begin": "<", "captures": { "0": { "name": "constant.other.symbol.ruby" } }, "end": ">", "patterns": [ { "include": "#angles" }, { "include": "#symbol" } ] } ] }, "braces": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\\\{|\\\\\\}", "name": "constant.other.symbol.ruby" }, { "begin": "\\{", "captures": { "0": { "name": "constant.other.symbol.ruby" } }, "end": "\\}", "patterns": [ { "include": "#braces" }, { "include": "#symbol" } ] } ] }, "brackets": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\\\[|\\\\\\]", "name": "constant.other.symbol.ruby" }, { "begin": "\\[", "captures": { "0": { "name": "constant.other.symbol.ruby" } }, "end": "\\]", "patterns": [ { "include": "#brackets" }, { "include": "#symbol" } ] } ] }, "parens": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\\\(|\\\\\\)", "name": "constant.other.symbol.ruby" }, { "begin": "\\(", "captures": { "0": { "name": "constant.other.symbol.ruby" } }, "end": "\\)", "patterns": [ { "include": "#parens" }, { "include": "#symbol" } ] } ] }, "symbol": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\\\\\|\\\\[ ]", "name": "constant.other.symbol.ruby" }, { "match": "\\S\\w*", "name": "constant.other.symbol.ruby" } ] } } }, { "begin": "%I(?:([(\\[{<])|([^\\w\\s]|_))", "beginCaptures": { "0": { "name": "punctuation.section.array.begin.ruby" } }, "end": "[)\\]}>]\\2|\\1\\2", "endCaptures": { "0": { "name": "punctuation.section.array.end.ruby" } }, "name": "meta.array.symbol.interpolated.ruby", "patterns": [ { "begin": "\\G(?<=\\()(?!\\))", "end": "(?=\\))", "patterns": [ { "include": "#parens" }, { "include": "#symbol" } ] }, { "begin": "\\G(?<=\\[)(?!\\])", "end": "(?=\\])", "patterns": [ { "include": "#brackets" }, { "include": "#symbol" } ] }, { "begin": "\\G(?<=\\{)(?!\\})", "end": "(?=\\})", "patterns": [ { "include": "#braces" }, { "include": "#symbol" } ] }, { "begin": "\\G(?<=<)(?!>)", "end": "(?=>)", "patterns": [ { "include": "#angles" }, { "include": "#symbol" } ] }, { "include": "#symbol" } ], "repository": { "angles": { "patterns": [ { "begin": "<", "captures": { "0": { "name": "constant.other.symbol.ruby" } }, "end": ">", "patterns": [ { "include": "#angles" }, { "include": "#symbol" } ] } ] }, "braces": { "patterns": [ { "begin": "\\{", "captures": { "0": { "name": "constant.other.symbol.ruby" } }, "end": "\\}", "patterns": [ { "include": "#braces" }, { "include": "#symbol" } ] } ] }, "brackets": { "patterns": [ { "begin": "\\[", "captures": { "0": { "name": "constant.other.symbol.ruby" } }, "end": "\\]", "patterns": [ { "include": "#brackets" }, { "include": "#symbol" } ] } ] }, "parens": { "patterns": [ { "begin": "\\(", "captures": { "0": { "name": "constant.other.symbol.ruby" } }, "end": "\\)", "patterns": [ { "include": "#parens" }, { "include": "#symbol" } ] } ] }, "symbol": { "patterns": [ { "begin": "(?=\\\\|#\\{)", "end": "(?!\\G)", "name": "constant.other.symbol.ruby", "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" } ] }, { "match": "\\S\\w*", "name": "constant.other.symbol.ruby" } ] } } }, { "begin": "%q(?:([(\\[{<])|([^\\w\\s]|_))", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "end": "[)\\]}>]\\2|\\1\\2", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.quoted.other.ruby", "patterns": [ { "begin": "\\G(?<=\\()(?!\\))", "end": "(?=\\))", "patterns": [ { "include": "#parens" } ] }, { "begin": "\\G(?<=\\[)(?!\\])", "end": "(?=\\])", "patterns": [ { "include": "#brackets" } ] }, { "begin": "\\G(?<=\\{)(?!\\})", "end": "(?=\\})", "patterns": [ { "include": "#braces" } ] }, { "begin": "\\G(?<=<)(?!>)", "end": "(?=>)", "patterns": [ { "include": "#angles" } ] } ], "repository": { "angles": { "patterns": [ { "match": "\\\\<|\\\\>|\\\\\\\\", "name": "constant.character.escape.ruby" }, { "begin": "<", "end": ">", "patterns": [ { "include": "#angles" } ] } ] }, "braces": { "patterns": [ { "match": "\\\\\\{|\\\\\\}|\\\\\\\\", "name": "constant.character.escape.ruby" }, { "begin": "\\{", "end": "\\}", "patterns": [ { "include": "#braces" } ] } ] }, "brackets": { "patterns": [ { "match": "\\\\\\[|\\\\\\]|\\\\\\\\", "name": "constant.character.escape.ruby" }, { "begin": "\\[", "end": "\\]", "patterns": [ { "include": "#brackets" } ] } ] }, "parens": { "patterns": [ { "match": "\\\\\\(|\\\\\\)|\\\\\\\\", "name": "constant.character.escape.ruby" }, { "begin": "\\(", "end": "\\)", "patterns": [ { "include": "#parens" } ] } ] } } }, { "begin": "%Q?(?:([(\\[{<])|([^\\w\\s=]|_))", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "end": "[)\\]}>]\\2|\\1\\2", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.quoted.other.interpolated.ruby", "patterns": [ { "begin": "\\G(?<=\\()(?!\\))", "end": "(?=\\))", "patterns": [ { "include": "#parens" } ] }, { "begin": "\\G(?<=\\[)(?!\\])", "end": "(?=\\])", "patterns": [ { "include": "#brackets" } ] }, { "begin": "\\G(?<=\\{)(?!\\})", "end": "(?=\\})", "patterns": [ { "include": "#braces" } ] }, { "begin": "\\G(?<=<)(?!>)", "end": "(?=>)", "patterns": [ { "include": "#angles" } ] }, { "include": "#escaped_char" }, { "include": "#interpolated_ruby" } ], "repository": { "angles": { "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" }, { "begin": "<", "end": ">", "patterns": [ { "include": "#angles" } ] } ] }, "braces": { "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" }, { "begin": "\\{", "end": "\\}", "patterns": [ { "include": "#braces" } ] } ] }, "brackets": { "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" }, { "begin": "\\[", "end": "\\]", "patterns": [ { "include": "#brackets" } ] } ] }, "parens": { "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" }, { "begin": "\\(", "end": "\\)", "patterns": [ { "include": "#parens" } ] } ] } } }, { "begin": "%r(?:([(\\[{<])|([^\\w\\s]|_))", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "end": "([)\\]}>]\\2|\\1\\2)[eimnosux]*", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.regexp.percent.ruby", "patterns": [ { "begin": "\\G(?<=\\()(?!\\))", "end": "(?=\\))", "patterns": [ { "include": "#parens" } ] }, { "begin": "\\G(?<=\\[)(?!\\])", "end": "(?=\\])", "patterns": [ { "include": "#brackets" } ] }, { "begin": "\\G(?<=\\{)(?!\\})", "end": "(?=\\})", "patterns": [ { "include": "#braces" } ] }, { "begin": "\\G(?<=<)(?!>)", "end": "(?=>)", "patterns": [ { "include": "#angles" } ] }, { "include": "#regex_sub" } ], "repository": { "angles": { "patterns": [ { "include": "#regex_sub" }, { "begin": "<", "end": ">", "patterns": [ { "include": "#angles" } ] } ] }, "braces": { "patterns": [ { "include": "#regex_sub" }, { "begin": "\\{", "end": "\\}", "patterns": [ { "include": "#braces" } ] } ] }, "brackets": { "patterns": [ { "include": "#regex_sub" }, { "begin": "\\[", "end": "\\]", "patterns": [ { "include": "#brackets" } ] } ] }, "parens": { "patterns": [ { "include": "#regex_sub" }, { "begin": "\\(", "end": "\\)", "patterns": [ { "include": "#parens" } ] } ] } } }, { "begin": "%s(?:([(\\[{<])|([^\\w\\s]|_))", "beginCaptures": { "0": { "name": "punctuation.definition.constant.begin.ruby" } }, "end": "[)\\]}>]\\2|\\1\\2", "endCaptures": { "0": { "name": "punctuation.definition.constant.end.ruby" } }, "name": "constant.other.symbol.percent.ruby", "patterns": [ { "begin": "\\G(?<=\\()(?!\\))", "end": "(?=\\))", "patterns": [ { "include": "#parens" } ] }, { "begin": "\\G(?<=\\[)(?!\\])", "end": "(?=\\])", "patterns": [ { "include": "#brackets" } ] }, { "begin": "\\G(?<=\\{)(?!\\})", "end": "(?=\\})", "patterns": [ { "include": "#braces" } ] }, { "begin": "\\G(?<=<)(?!>)", "end": "(?=>)", "patterns": [ { "include": "#angles" } ] } ], "repository": { "angles": { "patterns": [ { "match": "\\\\<|\\\\>|\\\\\\\\", "name": "constant.character.escape.ruby" }, { "begin": "<", "end": ">", "patterns": [ { "include": "#angles" } ] } ] }, "braces": { "patterns": [ { "match": "\\\\\\{|\\\\\\}|\\\\\\\\", "name": "constant.character.escape.ruby" }, { "begin": "\\{", "end": "\\}", "patterns": [ { "include": "#braces" } ] } ] }, "brackets": { "patterns": [ { "match": "\\\\\\[|\\\\\\]|\\\\\\\\", "name": "constant.character.escape.ruby" }, { "begin": "\\[", "end": "\\]", "patterns": [ { "include": "#brackets" } ] } ] }, "parens": { "patterns": [ { "match": "\\\\\\(|\\\\\\)|\\\\\\\\", "name": "constant.character.escape.ruby" }, { "begin": "\\(", "end": "\\)", "patterns": [ { "include": "#parens" } ] } ] } } }, { "begin": "%w(?:([(\\[{<])|([^\\w\\s]|_))", "beginCaptures": { "0": { "name": "punctuation.section.array.begin.ruby" } }, "end": "[)\\]}>]\\2|\\1\\2", "endCaptures": { "0": { "name": "punctuation.section.array.end.ruby" } }, "name": "meta.array.string.ruby", "patterns": [ { "begin": "\\G(?<=\\()(?!\\))", "end": "(?=\\))", "patterns": [ { "include": "#parens" }, { "include": "#string" } ] }, { "begin": "\\G(?<=\\[)(?!\\])", "end": "(?=\\])", "patterns": [ { "include": "#brackets" }, { "include": "#string" } ] }, { "begin": "\\G(?<=\\{)(?!\\})", "end": "(?=\\})", "patterns": [ { "include": "#braces" }, { "include": "#string" } ] }, { "begin": "\\G(?<=<)(?!>)", "end": "(?=>)", "patterns": [ { "include": "#angles" }, { "include": "#string" } ] }, { "include": "#string" } ], "repository": { "angles": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\<|\\\\>", "name": "string.other.ruby" }, { "begin": "<", "captures": { "0": { "name": "string.other.ruby" } }, "end": ">", "patterns": [ { "include": "#angles" }, { "include": "#string" } ] } ] }, "braces": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\\\{|\\\\\\}", "name": "string.other.ruby" }, { "begin": "\\{", "captures": { "0": { "name": "string.other.ruby" } }, "end": "\\}", "patterns": [ { "include": "#braces" }, { "include": "#string" } ] } ] }, "brackets": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\\\[|\\\\\\]", "name": "string.other.ruby" }, { "begin": "\\[", "captures": { "0": { "name": "string.other.ruby" } }, "end": "\\]", "patterns": [ { "include": "#brackets" }, { "include": "#string" } ] } ] }, "parens": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\\\(|\\\\\\)", "name": "string.other.ruby" }, { "begin": "\\(", "captures": { "0": { "name": "string.other.ruby" } }, "end": "\\)", "patterns": [ { "include": "#parens" }, { "include": "#string" } ] } ] }, "string": { "patterns": [ { "captures": { "0": { "name": "constant.character.escape.ruby" } }, "match": "\\\\\\\\|\\\\[ ]", "name": "string.other.ruby" }, { "match": "\\S\\w*", "name": "string.other.ruby" } ] } } }, { "begin": "%W(?:([(\\[{<])|([^\\w\\s]|_))", "beginCaptures": { "0": { "name": "punctuation.section.array.begin.ruby" } }, "end": "[)\\]}>]\\2|\\1\\2", "endCaptures": { "0": { "name": "punctuation.section.array.end.ruby" } }, "name": "meta.array.string.interpolated.ruby", "patterns": [ { "begin": "\\G(?<=\\()(?!\\))", "end": "(?=\\))", "patterns": [ { "include": "#parens" }, { "include": "#string" } ] }, { "begin": "\\G(?<=\\[)(?!\\])", "end": "(?=\\])", "patterns": [ { "include": "#brackets" }, { "include": "#string" } ] }, { "begin": "\\G(?<=\\{)(?!\\})", "end": "(?=\\})", "patterns": [ { "include": "#braces" }, { "include": "#string" } ] }, { "begin": "\\G(?<=<)(?!>)", "end": "(?=>)", "patterns": [ { "include": "#angles" }, { "include": "#string" } ] }, { "include": "#string" } ], "repository": { "angles": { "patterns": [ { "begin": "<", "captures": { "0": { "name": "string.other.ruby" } }, "end": ">", "patterns": [ { "include": "#angles" }, { "include": "#string" } ] } ] }, "braces": { "patterns": [ { "begin": "\\{", "captures": { "0": { "name": "string.other.ruby" } }, "end": "\\}", "patterns": [ { "include": "#braces" }, { "include": "#string" } ] } ] }, "brackets": { "patterns": [ { "begin": "\\[", "captures": { "0": { "name": "string.other.ruby" } }, "end": "\\]", "patterns": [ { "include": "#brackets" }, { "include": "#string" } ] } ] }, "parens": { "patterns": [ { "begin": "\\(", "captures": { "0": { "name": "string.other.ruby" } }, "end": "\\)", "patterns": [ { "include": "#parens" }, { "include": "#string" } ] } ] }, "string": { "patterns": [ { "begin": "(?=\\\\|#\\{)", "end": "(?!\\G)", "name": "string.other.ruby", "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" } ] }, { "match": "\\S\\w*", "name": "string.other.ruby" } ] } } }, { "begin": "%x(?:([(\\[{<])|([^\\w\\s]|_))", "beginCaptures": { "0": { "name": "punctuation.definition.string.begin.ruby" } }, "end": "[)\\]}>]\\2|\\1\\2", "endCaptures": { "0": { "name": "punctuation.definition.string.end.ruby" } }, "name": "string.interpolated.percent.ruby", "patterns": [ { "begin": "\\G(?<=\\()(?!\\))", "end": "(?=\\))", "patterns": [ { "include": "#parens" } ] }, { "begin": "\\G(?<=\\[)(?!\\])", "end": "(?=\\])", "patterns": [ { "include": "#brackets" } ] }, { "begin": "\\G(?<=\\{)(?!\\})", "end": "(?=\\})", "patterns": [ { "include": "#braces" } ] }, { "begin": "\\G(?<=<)(?!>)", "end": "(?=>)", "patterns": [ { "include": "#angles" } ] }, { "include": "#escaped_char" }, { "include": "#interpolated_ruby" } ], "repository": { "angles": { "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" }, { "begin": "<", "end": ">", "patterns": [ { "include": "#angles" } ] } ] }, "braces": { "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" }, { "begin": "\\{", "end": "\\}", "patterns": [ { "include": "#braces" } ] } ] }, "brackets": { "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" }, { "begin": "\\[", "end": "\\]", "patterns": [ { "include": "#brackets" } ] } ] }, "parens": { "patterns": [ { "include": "#escaped_char" }, { "include": "#interpolated_ruby" }, { "begin": "\\(", "end": "\\)", "patterns": [ { "include": "#parens" } ] } ] } } } ] }, "regex_sub": { "patterns": [ { "include": "#interpolated_ruby" }, { "include": "#escaped_char" }, { "captures": { "1": { "name": "punctuation.definition.quantifier.begin.ruby" }, "3": { "name": "punctuation.definition.quantifier.end.ruby" } }, "match": "(\\{)\\d+(,\\d+)?(\\})", "name": "keyword.operator.quantifier.ruby" }, { "begin": "\\[\\^?", "beginCaptures": { "0": { "name": "punctuation.definition.character-class.begin.ruby" } }, "end": "\\]", "endCaptures": { "0": { "name": "punctuation.definition.character-class.end.ruby" } }, "name": "constant.other.character-class.set.ruby", "patterns": [ { "include": "#escaped_char" } ] }, { "begin": "\\(\\?#", "beginCaptures": { "0": { "name": "punctuation.definition.comment.begin.ruby" } }, "end": "\\)", "endCaptures": { "0": { "name": "punctuation.definition.comment.end.ruby" } }, "name": "comment.line.number-sign.ruby", "patterns": [ { "include": "#escaped_char" } ] }, { "begin": "\\(", "captures": { "0": { "name": "punctuation.definition.group.ruby" } }, "end": "\\)", "name": "meta.group.regexp.ruby", "patterns": [ { "include": "#regex_sub" } ] }, { "begin": "(?<=^|\\s)(#)\\s(?=[[a-zA-Z0-9,. \\t?!-][^\\x{00}-\\x{7F}]]*$)", "beginCaptures": { "1": { "name": "punctuation.definition.comment.ruby" } }, "comment": "We are restrictive in what we allow to go after the comment character to avoid false positives, since the availability of comments depend on regexp flags.", "end": "$\\n?", "name": "comment.line.number-sign.ruby" } ] } } } ================================================ FILE: textmate/rust.tmLanguage.json ================================================ { "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", "version": "https://github.com/zargony/atom-language-rust/commit/5238d9834953ed7c58d9b5b9bb0c084c3c11ecd6", "name": "Rust", "scopeName": "source.rust", "patterns": [ { "comment": "Keywords that have a different meaning when used at top-level", "match": "\\b(for)\\b", "name": "keyword.other.top.rust" }, { "include": "#code" } ], "repository": { "code": { "patterns": [ { "comment": "By entering a block, we stop matching any top-level patterns that aren't inside #code", "begin": "{", "end": "}", "name": "meta.block.rust", "patterns": [ { "include": "#code" } ] }, { "include": "#comments" }, { "include": "#attribute" }, { "include": "#keywords" }, { "include": "#literals" } ] }, "attribute": { "patterns": [ { "comment": "Attribute", "name": "meta.attribute.rust", "begin": "#\\!?\\[", "end": "\\]", "beginCaptures": { "0": { "name": "punctuation.definition.tag.attribute.begin.rust" } }, "endCaptures": { "0": { "name": "punctuation.definition.tag.attribute.end.rust" } }, "patterns": [ { "include": "#metaItem" } ] } ] }, "metaItem": { "patterns": [ { "begin": "\\b\\w+\\(", "end": "\\)", "beginCaptures": { "0": { "name": "punctuation.definition.tag.attribute.metaItem.begin.rust" } }, "endCaptures": { "0": { "name": "punctuation.definition.tag.attribute.metaItem.end.rust" } }, "patterns": [ { "include": "#metaItem" } ] }, { "match": "\\b\\w+\\s*=", "name": "punctuation.definition.tag.attribute.metaItem.set.rust" }, { "include": "#literals" }, { "include": "#comments" } ] }, "keywords": { "patterns": [ { "comment": "Regular keywords", "match": "\\b(async|as|'static|Self|abstract|box|const|crate|dyn|enum|extern|final|fn|impl|let|macro|mod|mut|override|priv|pub|ref|self|static|struct|super|trait|type|union|unsized|use|virtual|where)\\b", "name": "keyword.other.rust" }, { "comment": "Control keywords", "match": "\\b(await|become|break|continue|do|else|for|if|in|loop|match|move|return|try|typeof|unsafe|while|yield)\\b", "name": "keyword.control.rust" }, { "comment": "Miscellaneous operator", "name": "keyword.operator.misc.rust", "match": "(=>|::)" }, { "comment": "Comparison operator", "name": "keyword.operator.comparison.rust", "match": "(&&|\\|\\||==|!=)" }, { "comment": "Assignment operator", "name": "keyword.operator.assignment.rust", "match": "(\\+=|-=|/=|\\*=|%=|\\^=|&=|\\|=|<<=|>>=|=)" }, { "comment": "Arithmetic operator", "name": "keyword.operator.arithmetic.rust", "match": "(!|\\+|-|/|\\*|%|\\^|&|\\||<<|>>)" }, { "comment": "Sigil", "name": "keyword.operator.sigil.rust", "match": "[&*](?=[a-zA-Z0-9_\\(\\[\\|\\\"]+)" }, { "comment": "Comparison operator (second group because of regex precedence)", "name": "keyword.operator.comparison.rust", "match": "(<=|>=|<|>)" }, { "comment": "Terminator", "match": ";", "name": "keyword.other.semi.go" } ] }, "literals": { "patterns": [ { "comment": "Boolean literals", "match": "\\b(true|false)\\b", "name": "constant.numeric.boolean.rust" }, { "comment": "Floating point literal (fraction)", "name": "constant.numeric.float.rust", "match": "\\b[0-9][0-9_]*\\.[0-9][0-9_]*([eE][+-]?[0-9_]+)?(f32|f64)?\\b" }, { "comment": "Floating point literal (exponent)", "name": "constant.numeric.float.rust", "match": "\\b[0-9][0-9_]*(\\.[0-9][0-9_]*)?[eE][+-]?[0-9_]+(f32|f64)?\\b" }, { "comment": "Floating point literal (typed)", "name": "constant.numeric.float.rust", "match": "\\b[0-9][0-9_]*(\\.[0-9][0-9_]*)?([eE][+-]?[0-9_]+)?(f32|f64)\\b" }, { "comment": "Integer literal (decimal)", "name": "constant.numeric.integer.decimal.rust", "match": "\\b[0-9][0-9_]*([ui](8|16|32|64|128|s|size))?\\b" }, { "comment": "Integer literal (hexadecimal)", "name": "constant.numeric.integer.hexadecimal.rust", "match": "\\b0x[a-fA-F0-9_]+([ui](8|16|32|64|128|s|size))?\\b" }, { "comment": "Integer literal (octal)", "name": "constant.numeric.integer.octal.rust", "match": "\\b0o[0-7_]+([ui](8|16|32|64|128|s|size))?\\b" }, { "comment": "Integer literal (binary)", "name": "constant.numeric.integer.binary.rust", "match": "\\b0b[01_]+([ui](8|16|32|64|128|s|size))?\\b" }, { "comment": "Single-quote string literal (character)", "name": "string.quoted.single.rust", "match": "b?'([^'\\\\]|\\\\(x[0-9A-Fa-f]{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.))'" }, { "comment": "Double-quote string literal", "name": "string.quoted.double.rust", "begin": "b?\"", "end": "\"", "patterns": [ { "include": "#escaped_character" } ] }, { "comment": "Raw double-quote string literal", "name": "string.quoted.double.raw.rust", "begin": "b?r(#*)\"", "end": "\"\\1" } ] }, "comments": { "patterns": [ { "include": "#block_doc_comment" }, { "include": "#block_comment" }, { "comment": "Single-line documentation comment", "name": "comment.line.documentation.rust", "begin": "//[!/](?=[^/])", "end": "$" }, { "comment": "Single-line comment", "name": "comment.line.double-slash.rust", "begin": "//", "end": "$" } ] }, "block_doc_comment": { "patterns": [ { "comment": "Block documentation comment", "name": "comment.block.documentation.rust", "begin": "/\\*[\\*!](?![\\*/])", "end": "\\*/", "patterns": [ { "include": "#block_doc_comment" }, { "include": "#block_comment" } ] } ] }, "block_comment": { "patterns": [ { "comment": "Block comment", "name": "comment.block.rust", "begin": "/\\*", "end": "\\*/", "patterns": [ { "include": "#block_doc_comment" }, { "include": "#block_comment" } ] } ] }, "escaped_character": { "patterns": [ { "name": "constant.character.escape.rust", "match": "\\\\(x[0-9A-Fa-f]{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.)" } ] } } } ================================================ FILE: textmate/typescript.tmLanguage.json ================================================ { "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", "name": "TypeScript", "scopeName": "source.ts", "fileTypes": [ "ts", "js" ], "patterns": [ { "include": "#expression" } ], "repository": { "expression": { "patterns": [ { "include": "#keyword" }, { "include": "#decorator" }, { "include": "#regex" }, { "include": "#string" }, { "include": "#template" }, { "include": "#literal" }, { "include": "#comment" } ] }, "keyword": { "patterns": [ { "match": "\\b(module)\\s+(?=[\\w'\"])", "captures": { "1": { "name": "keyword.other.ts" } } }, { "match": "\\b(declare|namespace|interface|type)\\s+(?=\\w)", "captures": { "1": { "name": "keyword.other.ts" } } }, { "match": "\\b(abstract|arguments|as|async|class|const|enum|export|extends|from|function|implements|import|let|package|private|protected|public|static|super|this|var|void|with)\\b", "name": "keyword.other.ts" }, { "match": "\\b(await|break|case|catch|continue|debugger|default|delete|do|in|of|else|eval|finally|for|if|instanceof|new|return|switch|throw|try|typeof|while|yield)\\b", "name": "keyword.control.ts" }, { "comment": "Terminator", "match": ";", "name": "keyword.other.semi.ts" } ] }, "decorator": { "name": "meta.decorator.ts", "begin": "(?|&&|\\|\\||\\*\\/)\\s*(/)(?![/*+?])(?=.*/)", "beginCaptures": { }, "end": "(/)([gimuy]*)", "endCaptures": { "2": { "name": "keyword.other.ts" } }, "patterns": [ { "include": "#regexp" } ] }, { "name": "string.regex.ts", "begin": "/(?![/*])(?=(?:[^/\\\\\\[]|\\\\.|\\[([^\\]\\\\]|\\\\.)+\\])+/(?![/*])[gimy]*(?!\\s*[a-zA-Z0-9_$]))", "beginCaptures": { }, "end": "(/)([gimuy]*)", "endCaptures": { "2": { "name": "keyword.other.ts" } }, "patterns": [ { "include": "#regexp" } ] } ] }, "regexp": { "patterns": [ { "name": "keyword.control.anchor.regexp", "match": "\\\\[bB]|\\^|\\$" }, { "name": "keyword.other.back-reference.regexp", "match": "\\\\[1-9]\\d*" }, { "name": "keyword.operator.quantifier.regexp", "match": "[?+*]|\\{(\\d+,\\d+|\\d+,|,\\d+|\\d+)\\}\\??" }, { "name": "keyword.operator.or.regexp", "match": "\\|" }, { "name": "meta.group.assertion.regexp", "begin": "(\\()((\\?=)|(\\?!))", "beginCaptures": { "3": { "name": "meta.assertion.look-ahead.regexp" }, "4": { "name": "meta.assertion.negative-look-ahead.regexp" } }, "end": "(\\))", "endCaptures": { }, "patterns": [ { "include": "#regexp" } ] }, { "name": "meta.group.regexp", "begin": "\\((\\?:)?", "beginCaptures": { }, "end": "\\)", "endCaptures": { }, "patterns": [ { "include": "#regexp" } ] }, { "name": "constant.other.character-class.set.regexp", "begin": "(\\[)(\\^)?", "beginCaptures": { "2": { "name": "keyword.operator.negation.regexp" } }, "end": "(\\])", "endCaptures": { }, "patterns": [ { "name": "constant.other.character-class.range.regexp", "match": "(?:.|(\\\\(?:[0-7]{3}|x\\h\\h|u\\h\\h\\h\\h))|(\\\\c[A-Z])|(\\\\.))\\-(?:[^\\]\\\\]|(\\\\(?:[0-7]{3}|x\\h\\h|u\\h\\h\\h\\h))|(\\\\c[A-Z])|(\\\\.))", "captures": { "1": { "name": "constant.character.numeric.regexp" }, "2": { "name": "constant.character.control.regexp" }, "3": { "name": "constant.character.escape.backslash.regexp" }, "4": { "name": "constant.character.numeric.regexp" }, "5": { "name": "constant.character.control.regexp" }, "6": { "name": "constant.character.escape.backslash.regexp" } } }, { "include": "#regex-character-class" } ] }, { "include": "#regex-character-class" } ] }, "regex-character-class": { "patterns": [ { "name": "constant.other.character-class.regexp", "match": "\\\\[wWsSdDtrnvf]|\\." }, { "name": "constant.character.numeric.regexp", "match": "\\\\([0-7]{3}|x\\h\\h|u\\h\\h\\h\\h)" }, { "name": "constant.character.control.regexp", "match": "\\\\c[A-Z]" }, { "name": "constant.character.escape.backslash.regexp", "match": "\\\\." } ] }, "string": { "name": "string.ts", "patterns": [ { "include": "#qstring-single" }, { "include": "#qstring-double" } ] }, "template": { "name": "string.template.ts", "begin": "([_$[:alpha:]][_$[:alnum:]]*)?(`)", "beginCaptures": { "1": { "name": "entity.name.function.tagged-template.ts" } }, "end": "`", "patterns": [ { "include": "#template-substitution-element" }, { "include": "#string-character-escape" } ] }, "string-character-escape": { "name": "constant.character.escape.ts", "match": "\\\\(x\\h{2}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.|$)" }, "template-substitution-element": { "name": "meta.template.expression.ts", "begin": "\\$\\{", "end": "\\}", "patterns": [ { "include": "#expression" } ] }, "literal": { "name": "literal.ts", "patterns": [ { "match": "\\b(true|false)\\b", "name": "constant.numeric.boolean.ts" }, { "match": "\\b(null|undefined)\\b", "name": "constant.numeric.null.ts" }, { "include": "#numeric-literal" }, { "include": "#undefined-literal" }, { "include": "#numericConstant-literal" } ] }, "numeric-literal": { "patterns": [ { "name": "constant.numeric.hex.ts", "match": "\\b(? # {Array} or {Object} type application (optional .)\n )\n (?:\n [\\.|~] # {Foo.bar} namespaced, {string|number} multiple, {Foo~bar} class-specific callback\n [a-zA-Z_$]+\n (?:\n (?:\n [\\w$]*\n (?:\\[\\])? # {(string|number[])} type application, a string or an array of numbers\n ) |\n \\.?<[\\w$]+(?:,\\s+[\\w$]+)*> # {Array} or {Object} type application (optional .)\n )\n )*\n \\) |\n [a-zA-Z_$]+\n (?:\n (?:\n [\\w$]*\n (?:\\[\\])? # {string[]|number} type application, an array of strings or a number\n ) |\n \\.?<[\\w$]+(?:,\\s+[\\w$]+)*> # {Array} or {Object} type application (optional .)\n )\n (?:\n [\\.|~] # {Foo.bar} namespaced, {string|number} multiple, {Foo~bar} class-specific callback\n [a-zA-Z_$]+\n (?:\n [\\w$]* |\n \\.?<[\\w$]+(?:,\\s+[\\w$]+)*> # {Array} or {Object} type application (optional .)\n )\n )*\n )\n # Check for suffix\n (?:\\[\\])? # {string[]} type application, an array of strings\n =? # {string=} optional parameter\n)})\n\\s+\n(\n \\[ # [foo] optional parameter\n \\s*\n (?:\n [a-zA-Z_$][\\w$]*\n (?:\n (?:\\[\\])? # Foo[].bar properties within an array\n \\. # Foo.Bar namespaced parameter\n [a-zA-Z_$][\\w$]*\n )*\n (?:\n \\s*\n = # [foo=bar] Default parameter value\n \\s*\n [\\w$\\s]*\n )?\n )\n \\s*\n \\] |\n (?:\n [a-zA-Z_$][\\w$]*\n (?:\n (?:\\[\\])? # Foo[].bar properties within an array\n \\. # Foo.Bar namespaced parameter\n [a-zA-Z_$][\\w$]*\n )*\n )?\n)\n\\s+\n(?:-\\s+)? # optional hyphen before the description\n((?:(?!\\*\\/).)*) # The type description", "captures": { "0": { "name": "other.meta.jsdoc" }, "2": { "name": "variable.other.jsdoc" }, "3": { "name": "other.description.jsdoc" } } }, { "match": "(?x)\n({(?:\n \\* | # {*} any type\n \\? | # {?} unknown type\n\n (?: # Check for a prefix\n \\? | # {?string} nullable type\n ! | # {!string} non-nullable type\n \\.{3} # {...string} variable number of parameters\n )?\n\n (?:\n \\( # Opening bracket of multiple types with parenthesis {(string|number)}\n [a-zA-Z_$]+\n (?:\n [\\w$]* |\n \\.?<[\\w$]+(?:,\\s+[\\w$]+)*> # {Array} or {Object} type application (optional .)\n )\n (?:\n [\\.|~] # {Foo.bar} namespaced, {string|number} multiple, {Foo~bar} class-specific callback\n [a-zA-Z_$]+\n (?:\n [\\w$]* |\n \\.?<[\\w$]+(?:,\\s+[\\w$]+)*> # {Array} or {Object} type application (optional .)\n )\n )*\n \\) |\n [a-zA-Z_$]+\n (?:\n [\\w$]* |\n \\.?<[\\w$]+(?:,\\s+[\\w$]+)*> # {Array} or {Object} type application (optional .)\n )\n (?:\n [\\.|~] # {Foo.bar} namespaced, {string|number} multiple, {Foo~bar} class-specific callback\n [a-zA-Z_$]+\n (?:\n [\\w$]* |\n \\.?<[\\w$]+(?:,\\s+[\\w$]+)*> # {Array} or {Object} type application (optional .)\n )\n )*\n )\n # Check for suffix\n (?:\\[\\])? # {string[]} type application, an array of strings\n =? # {string=} optional parameter\n)})\n\\s+\n(?:-\\s+)? # optional hyphen before the description\n((?:(?!\\*\\/).)*) # The type description", "captures": { "0": { "name": "other.meta.jsdoc" }, "1": { "name": "entity.name.type.instance.jsdoc" }, "2": { "name": "other.description.jsdoc" } } } ] } } } ================================================ FILE: tsconfig.json ================================================ { "compilerOptions": { "module": "commonjs", "target": "es6", "outDir": "out", "lib": [ "es6" ], "sourceMap": true, "rootDir": "src", /* Strict Type-Checking Option */ "strict": true, /* enable all strict type-checking options */ /* Additional Checks */ "noUnusedLocals": true /* Report errors on unused locals. */ // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ // "noUnusedParameters": true, /* Report errors on unused parameters. */ }, "exclude": [ "examples", "node_modules", ".vscode-test" ] } ================================================ FILE: tslint.json ================================================ { "rules": { "no-string-throw": true, "no-unused-expression": true, "no-duplicate-variable": true, "class-name": true, "semicolon": [false, "never"] }, "defaultSeverity": "warning" }